2024-12-03 23:14:26,586 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 2024-12-03 23:14:26,603 main DEBUG Took 0.014021 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-03 23:14:26,603 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-03 23:14:26,603 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-03 23:14:26,605 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-03 23:14:26,606 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 23:14:26,615 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-03 23:14:26,631 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 23:14:26,633 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 23:14:26,634 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 23:14:26,634 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 23:14:26,635 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 23:14:26,635 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 23:14:26,637 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 23:14:26,638 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 23:14:26,638 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 23:14:26,639 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 23:14:26,640 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 23:14:26,640 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 23:14:26,641 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 23:14:26,641 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 23:14:26,642 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 23:14:26,642 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 23:14:26,643 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 23:14:26,643 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 23:14:26,644 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 23:14:26,644 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 23:14:26,645 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 23:14:26,645 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 23:14:26,646 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 23:14:26,646 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 23:14:26,646 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 23:14:26,647 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-03 23:14:26,649 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 23:14:26,650 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-03 23:14:26,652 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-03 23:14:26,653 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-03 23:14:26,654 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-03 23:14:26,655 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-03 23:14:26,665 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-03 23:14:26,670 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-03 23:14:26,672 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-03 23:14:26,672 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-03 23:14:26,673 main DEBUG createAppenders(={Console}) 2024-12-03 23:14:26,674 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 initialized 2024-12-03 23:14:26,675 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 2024-12-03 23:14:26,675 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 OK. 2024-12-03 23:14:26,676 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-03 23:14:26,676 main DEBUG OutputStream closed 2024-12-03 23:14:26,676 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-03 23:14:26,677 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-03 23:14:26,677 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@c88a337 OK 2024-12-03 23:14:26,778 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-03 23:14:26,781 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-03 23:14:26,782 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-03 23:14:26,783 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-03 23:14:26,784 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-03 23:14:26,785 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-03 23:14:26,785 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-03 23:14:26,786 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-03 23:14:26,786 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-03 23:14:26,786 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-03 23:14:26,787 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-03 23:14:26,787 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-03 23:14:26,787 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-03 23:14:26,788 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-03 23:14:26,788 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-03 23:14:26,788 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-03 23:14:26,788 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-03 23:14:26,789 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-03 23:14:26,792 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-03 23:14:26,792 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@13330ac6) with optional ClassLoader: null 2024-12-03 23:14:26,793 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-03 23:14:26,794 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@13330ac6] started OK. 2024-12-03T23:14:26,811 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.master.assignment.TestRollbackSCP timeout: 13 mins 2024-12-03 23:14:26,815 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-03 23:14:26,815 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-03T23:14:27,126 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fb53328b-6122-8bbc-b1d5-834eeb8c019c 2024-12-03T23:14:27,156 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=class org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$HMasterForTest, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-03T23:14:27,182 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fb53328b-6122-8bbc-b1d5-834eeb8c019c/cluster_8abe0f93-9472-57fc-9787-96cccde33366, deleteOnExit=true 2024-12-03T23:14:27,183 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-03T23:14:27,185 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fb53328b-6122-8bbc-b1d5-834eeb8c019c/test.cache.data in system properties and HBase conf 2024-12-03T23:14:27,186 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fb53328b-6122-8bbc-b1d5-834eeb8c019c/hadoop.tmp.dir in system properties and HBase conf 2024-12-03T23:14:27,186 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fb53328b-6122-8bbc-b1d5-834eeb8c019c/hadoop.log.dir in system properties and HBase conf 2024-12-03T23:14:27,187 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fb53328b-6122-8bbc-b1d5-834eeb8c019c/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-03T23:14:27,188 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fb53328b-6122-8bbc-b1d5-834eeb8c019c/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-03T23:14:27,188 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-03T23:14:27,379 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-03T23:14:27,542 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-03T23:14:27,546 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fb53328b-6122-8bbc-b1d5-834eeb8c019c/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-03T23:14:27,547 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fb53328b-6122-8bbc-b1d5-834eeb8c019c/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-03T23:14:27,549 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fb53328b-6122-8bbc-b1d5-834eeb8c019c/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-03T23:14:27,550 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fb53328b-6122-8bbc-b1d5-834eeb8c019c/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-03T23:14:27,551 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fb53328b-6122-8bbc-b1d5-834eeb8c019c/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-03T23:14:27,551 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fb53328b-6122-8bbc-b1d5-834eeb8c019c/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-03T23:14:27,552 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fb53328b-6122-8bbc-b1d5-834eeb8c019c/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-03T23:14:27,552 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fb53328b-6122-8bbc-b1d5-834eeb8c019c/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-03T23:14:27,553 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fb53328b-6122-8bbc-b1d5-834eeb8c019c/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-03T23:14:27,553 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fb53328b-6122-8bbc-b1d5-834eeb8c019c/nfs.dump.dir in system properties and HBase conf 2024-12-03T23:14:27,554 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fb53328b-6122-8bbc-b1d5-834eeb8c019c/java.io.tmpdir in system properties and HBase conf 2024-12-03T23:14:27,555 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fb53328b-6122-8bbc-b1d5-834eeb8c019c/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-03T23:14:27,555 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fb53328b-6122-8bbc-b1d5-834eeb8c019c/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-03T23:14:27,563 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fb53328b-6122-8bbc-b1d5-834eeb8c019c/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-03T23:14:28,700 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-03T23:14:28,855 INFO [Time-limited test {}] log.Log(170): Logging initialized @3332ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-03T23:14:28,974 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-03T23:14:29,069 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-03T23:14:29,106 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-03T23:14:29,106 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-03T23:14:29,108 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-03T23:14:29,132 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-03T23:14:29,135 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3912242{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fb53328b-6122-8bbc-b1d5-834eeb8c019c/hadoop.log.dir/,AVAILABLE} 2024-12-03T23:14:29,136 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@595320c8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-03T23:14:29,467 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@64418798{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fb53328b-6122-8bbc-b1d5-834eeb8c019c/java.io.tmpdir/jetty-localhost-34717-hadoop-hdfs-3_4_1-tests_jar-_-any-18162034992006880913/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-03T23:14:29,476 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@11744524{HTTP/1.1, (http/1.1)}{localhost:34717} 2024-12-03T23:14:29,476 INFO [Time-limited test {}] server.Server(415): Started @3954ms 2024-12-03T23:14:29,953 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-03T23:14:29,962 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-03T23:14:29,963 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-03T23:14:29,963 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-03T23:14:29,964 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-03T23:14:29,969 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3a300759{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fb53328b-6122-8bbc-b1d5-834eeb8c019c/hadoop.log.dir/,AVAILABLE} 2024-12-03T23:14:29,970 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@474d518f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-03T23:14:30,135 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@79b30fb0{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fb53328b-6122-8bbc-b1d5-834eeb8c019c/java.io.tmpdir/jetty-localhost-37991-hadoop-hdfs-3_4_1-tests_jar-_-any-1680258697522392712/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-03T23:14:30,136 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@548cd725{HTTP/1.1, (http/1.1)}{localhost:37991} 2024-12-03T23:14:30,137 INFO [Time-limited test {}] server.Server(415): Started @4615ms 2024-12-03T23:14:30,203 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-03T23:14:30,372 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-03T23:14:30,381 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-03T23:14:30,386 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-03T23:14:30,386 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-03T23:14:30,387 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-03T23:14:30,388 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@29652cf4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fb53328b-6122-8bbc-b1d5-834eeb8c019c/hadoop.log.dir/,AVAILABLE} 2024-12-03T23:14:30,389 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1b0a32ba{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-03T23:14:30,572 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@34fcb585{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fb53328b-6122-8bbc-b1d5-834eeb8c019c/java.io.tmpdir/jetty-localhost-42115-hadoop-hdfs-3_4_1-tests_jar-_-any-7505983138098403309/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-03T23:14:30,573 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@389bb6f7{HTTP/1.1, (http/1.1)}{localhost:42115} 2024-12-03T23:14:30,574 INFO [Time-limited test {}] server.Server(415): Started @5052ms 2024-12-03T23:14:30,577 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-03T23:14:30,714 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-03T23:14:30,730 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-03T23:14:30,756 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-03T23:14:30,756 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-03T23:14:30,756 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-03T23:14:30,758 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@290e2eec{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fb53328b-6122-8bbc-b1d5-834eeb8c019c/hadoop.log.dir/,AVAILABLE} 2024-12-03T23:14:30,758 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@417efacb{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-03T23:14:30,844 WARN [Thread-103 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fb53328b-6122-8bbc-b1d5-834eeb8c019c/cluster_8abe0f93-9472-57fc-9787-96cccde33366/data/data1/current/BP-38275091-172.17.0.2-1733267668370/current, will proceed with Du for space computation calculation, 2024-12-03T23:14:30,844 WARN [Thread-104 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fb53328b-6122-8bbc-b1d5-834eeb8c019c/cluster_8abe0f93-9472-57fc-9787-96cccde33366/data/data2/current/BP-38275091-172.17.0.2-1733267668370/current, will proceed with Du for space computation calculation, 2024-12-03T23:14:30,884 WARN [Thread-109 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fb53328b-6122-8bbc-b1d5-834eeb8c019c/cluster_8abe0f93-9472-57fc-9787-96cccde33366/data/data3/current/BP-38275091-172.17.0.2-1733267668370/current, will proceed with Du for space computation calculation, 2024-12-03T23:14:30,889 WARN [Thread-110 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fb53328b-6122-8bbc-b1d5-834eeb8c019c/cluster_8abe0f93-9472-57fc-9787-96cccde33366/data/data4/current/BP-38275091-172.17.0.2-1733267668370/current, will proceed with Du for space computation calculation, 2024-12-03T23:14:30,936 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-03T23:14:30,946 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3c1f701{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fb53328b-6122-8bbc-b1d5-834eeb8c019c/java.io.tmpdir/jetty-localhost-44801-hadoop-hdfs-3_4_1-tests_jar-_-any-2488120343471975075/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-03T23:14:30,947 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3d599bb4{HTTP/1.1, (http/1.1)}{localhost:44801} 2024-12-03T23:14:30,948 INFO [Time-limited test {}] server.Server(415): Started @5426ms 2024-12-03T23:14:30,952 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-03T23:14:30,957 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-03T23:14:31,033 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x6c381984bf80a9a3 with lease ID 0xfd9becd3e0fa855f: Processing first storage report for DS-e710ddc2-08f6-4b71-b4db-e94cb5850278 from datanode DatanodeRegistration(127.0.0.1:44333, datanodeUuid=d3a8ad16-0743-44f9-9bd8-7ff859bebe67, infoPort=39967, infoSecurePort=0, ipcPort=46661, storageInfo=lv=-57;cid=testClusterID;nsid=1907083495;c=1733267668370) 2024-12-03T23:14:31,035 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x6c381984bf80a9a3 with lease ID 0xfd9becd3e0fa855f: from storage DS-e710ddc2-08f6-4b71-b4db-e94cb5850278 node DatanodeRegistration(127.0.0.1:44333, datanodeUuid=d3a8ad16-0743-44f9-9bd8-7ff859bebe67, infoPort=39967, infoSecurePort=0, ipcPort=46661, storageInfo=lv=-57;cid=testClusterID;nsid=1907083495;c=1733267668370), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-03T23:14:31,035 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x50c6f1ed617a1cbe with lease ID 0xfd9becd3e0fa855e: Processing first storage report for DS-91eef8b1-4e86-469d-90a0-bef7307996a0 from datanode DatanodeRegistration(127.0.0.1:38883, datanodeUuid=3e73d5b6-a866-4f43-8324-0c742cc48899, infoPort=37111, infoSecurePort=0, ipcPort=44581, storageInfo=lv=-57;cid=testClusterID;nsid=1907083495;c=1733267668370) 2024-12-03T23:14:31,035 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x50c6f1ed617a1cbe with lease ID 0xfd9becd3e0fa855e: from storage DS-91eef8b1-4e86-469d-90a0-bef7307996a0 node DatanodeRegistration(127.0.0.1:38883, datanodeUuid=3e73d5b6-a866-4f43-8324-0c742cc48899, infoPort=37111, infoSecurePort=0, ipcPort=44581, storageInfo=lv=-57;cid=testClusterID;nsid=1907083495;c=1733267668370), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-03T23:14:31,036 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x6c381984bf80a9a3 with lease ID 0xfd9becd3e0fa855f: Processing first storage report for DS-4b820811-3217-4ca7-bda0-b4842d107f19 from datanode DatanodeRegistration(127.0.0.1:44333, datanodeUuid=d3a8ad16-0743-44f9-9bd8-7ff859bebe67, infoPort=39967, infoSecurePort=0, ipcPort=46661, storageInfo=lv=-57;cid=testClusterID;nsid=1907083495;c=1733267668370) 2024-12-03T23:14:31,036 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x6c381984bf80a9a3 with lease ID 0xfd9becd3e0fa855f: from storage DS-4b820811-3217-4ca7-bda0-b4842d107f19 node DatanodeRegistration(127.0.0.1:44333, datanodeUuid=d3a8ad16-0743-44f9-9bd8-7ff859bebe67, infoPort=39967, infoSecurePort=0, ipcPort=46661, storageInfo=lv=-57;cid=testClusterID;nsid=1907083495;c=1733267668370), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-03T23:14:31,036 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x50c6f1ed617a1cbe with lease ID 0xfd9becd3e0fa855e: Processing first storage report for DS-63324289-3382-4714-bdc9-6115e13b76ea from datanode DatanodeRegistration(127.0.0.1:38883, datanodeUuid=3e73d5b6-a866-4f43-8324-0c742cc48899, infoPort=37111, infoSecurePort=0, ipcPort=44581, storageInfo=lv=-57;cid=testClusterID;nsid=1907083495;c=1733267668370) 2024-12-03T23:14:31,036 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x50c6f1ed617a1cbe with lease ID 0xfd9becd3e0fa855e: from storage DS-63324289-3382-4714-bdc9-6115e13b76ea node DatanodeRegistration(127.0.0.1:38883, datanodeUuid=3e73d5b6-a866-4f43-8324-0c742cc48899, infoPort=37111, infoSecurePort=0, ipcPort=44581, storageInfo=lv=-57;cid=testClusterID;nsid=1907083495;c=1733267668370), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-03T23:14:31,195 WARN [Thread-139 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fb53328b-6122-8bbc-b1d5-834eeb8c019c/cluster_8abe0f93-9472-57fc-9787-96cccde33366/data/data5/current/BP-38275091-172.17.0.2-1733267668370/current, will proceed with Du for space computation calculation, 2024-12-03T23:14:31,197 WARN [Thread-140 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fb53328b-6122-8bbc-b1d5-834eeb8c019c/cluster_8abe0f93-9472-57fc-9787-96cccde33366/data/data6/current/BP-38275091-172.17.0.2-1733267668370/current, will proceed with Du for space computation calculation, 2024-12-03T23:14:31,280 WARN [Thread-129 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-03T23:14:31,288 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe8955ef47c955602 with lease ID 0xfd9becd3e0fa8560: Processing first storage report for DS-e96a58d6-577c-4a7d-bf1d-88064ab38370 from datanode DatanodeRegistration(127.0.0.1:45029, datanodeUuid=6fb13a36-e8d9-4ed7-aafc-f06328038eb4, infoPort=34163, infoSecurePort=0, ipcPort=39381, storageInfo=lv=-57;cid=testClusterID;nsid=1907083495;c=1733267668370) 2024-12-03T23:14:31,288 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe8955ef47c955602 with lease ID 0xfd9becd3e0fa8560: from storage DS-e96a58d6-577c-4a7d-bf1d-88064ab38370 node DatanodeRegistration(127.0.0.1:45029, datanodeUuid=6fb13a36-e8d9-4ed7-aafc-f06328038eb4, infoPort=34163, infoSecurePort=0, ipcPort=39381, storageInfo=lv=-57;cid=testClusterID;nsid=1907083495;c=1733267668370), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-03T23:14:31,289 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe8955ef47c955602 with lease ID 0xfd9becd3e0fa8560: Processing first storage report for DS-a767b760-8558-49c1-9ff6-ae4c93b9d90f from datanode DatanodeRegistration(127.0.0.1:45029, datanodeUuid=6fb13a36-e8d9-4ed7-aafc-f06328038eb4, infoPort=34163, infoSecurePort=0, ipcPort=39381, storageInfo=lv=-57;cid=testClusterID;nsid=1907083495;c=1733267668370) 2024-12-03T23:14:31,289 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe8955ef47c955602 with lease ID 0xfd9becd3e0fa8560: from storage DS-a767b760-8558-49c1-9ff6-ae4c93b9d90f node DatanodeRegistration(127.0.0.1:45029, datanodeUuid=6fb13a36-e8d9-4ed7-aafc-f06328038eb4, infoPort=34163, infoSecurePort=0, ipcPort=39381, storageInfo=lv=-57;cid=testClusterID;nsid=1907083495;c=1733267668370), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-03T23:14:31,506 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fb53328b-6122-8bbc-b1d5-834eeb8c019c 2024-12-03T23:14:31,600 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fb53328b-6122-8bbc-b1d5-834eeb8c019c/cluster_8abe0f93-9472-57fc-9787-96cccde33366/zookeeper_0, clientPort=62825, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fb53328b-6122-8bbc-b1d5-834eeb8c019c/cluster_8abe0f93-9472-57fc-9787-96cccde33366/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fb53328b-6122-8bbc-b1d5-834eeb8c019c/cluster_8abe0f93-9472-57fc-9787-96cccde33366/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-03T23:14:31,613 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=62825 2024-12-03T23:14:31,626 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T23:14:31,629 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T23:14:31,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44333 is added to blk_1073741825_1001 (size=7) 2024-12-03T23:14:31,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45029 is added to blk_1073741825_1001 (size=7) 2024-12-03T23:14:31,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38883 is added to blk_1073741825_1001 (size=7) 2024-12-03T23:14:31,991 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1 with version=8 2024-12-03T23:14:31,992 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/hbase-staging 2024-12-03T23:14:32,110 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-03T23:14:32,417 INFO [Time-limited test {}] client.ConnectionUtils(128): master/5f9d64419733:0 server-side Connection retries=45 2024-12-03T23:14:32,431 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T23:14:32,432 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-03T23:14:32,438 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-03T23:14:32,438 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T23:14:32,438 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-03T23:14:32,607 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-03T23:14:32,680 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-03T23:14:32,691 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-03T23:14:32,696 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-03T23:14:32,729 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 7807 (auto-detected) 2024-12-03T23:14:32,731 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-12-03T23:14:32,760 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44117 2024-12-03T23:14:32,787 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:44117 connecting to ZooKeeper ensemble=127.0.0.1:62825 2024-12-03T23:14:32,830 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:441170x0, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-03T23:14:32,835 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:44117-0x1017a9bb4a60000 connected 2024-12-03T23:14:32,887 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T23:14:32,889 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T23:14:32,905 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44117-0x1017a9bb4a60000, quorum=127.0.0.1:62825, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-03T23:14:32,910 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1, hbase.cluster.distributed=false 2024-12-03T23:14:32,942 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44117-0x1017a9bb4a60000, quorum=127.0.0.1:62825, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-03T23:14:32,958 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44117 2024-12-03T23:14:32,960 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44117 2024-12-03T23:14:32,962 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44117 2024-12-03T23:14:32,968 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44117 2024-12-03T23:14:32,972 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44117 2024-12-03T23:14:33,116 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/5f9d64419733:0 server-side Connection retries=45 2024-12-03T23:14:33,118 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T23:14:33,119 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-03T23:14:33,119 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-03T23:14:33,119 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T23:14:33,119 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-03T23:14:33,123 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-03T23:14:33,125 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-03T23:14:33,126 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46833 2024-12-03T23:14:33,128 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:46833 connecting to ZooKeeper ensemble=127.0.0.1:62825 2024-12-03T23:14:33,130 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T23:14:33,134 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T23:14:33,144 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:468330x0, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-03T23:14:33,145 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:468330x0, quorum=127.0.0.1:62825, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-03T23:14:33,146 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:46833-0x1017a9bb4a60001 connected 2024-12-03T23:14:33,151 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-03T23:14:33,164 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-03T23:14:33,167 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46833-0x1017a9bb4a60001, quorum=127.0.0.1:62825, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-03T23:14:33,174 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46833-0x1017a9bb4a60001, quorum=127.0.0.1:62825, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-03T23:14:33,178 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46833 2024-12-03T23:14:33,178 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46833 2024-12-03T23:14:33,182 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46833 2024-12-03T23:14:33,183 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46833 2024-12-03T23:14:33,184 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46833 2024-12-03T23:14:33,203 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/5f9d64419733:0 server-side Connection retries=45 2024-12-03T23:14:33,203 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T23:14:33,203 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-03T23:14:33,204 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-03T23:14:33,204 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T23:14:33,204 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-03T23:14:33,204 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-03T23:14:33,205 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-03T23:14:33,206 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44899 2024-12-03T23:14:33,207 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:44899 connecting to ZooKeeper ensemble=127.0.0.1:62825 2024-12-03T23:14:33,209 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T23:14:33,212 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T23:14:33,220 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:448990x0, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-03T23:14:33,221 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:44899-0x1017a9bb4a60002 connected 2024-12-03T23:14:33,224 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44899-0x1017a9bb4a60002, quorum=127.0.0.1:62825, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-03T23:14:33,225 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-03T23:14:33,231 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-03T23:14:33,233 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44899-0x1017a9bb4a60002, quorum=127.0.0.1:62825, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-03T23:14:33,236 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44899-0x1017a9bb4a60002, quorum=127.0.0.1:62825, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-03T23:14:33,244 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44899 2024-12-03T23:14:33,245 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44899 2024-12-03T23:14:33,245 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44899 2024-12-03T23:14:33,248 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44899 2024-12-03T23:14:33,249 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44899 2024-12-03T23:14:33,271 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/5f9d64419733:0 server-side Connection retries=45 2024-12-03T23:14:33,271 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T23:14:33,271 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-03T23:14:33,272 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-03T23:14:33,272 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T23:14:33,272 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-03T23:14:33,272 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-03T23:14:33,272 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-03T23:14:33,286 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41167 2024-12-03T23:14:33,287 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:41167 connecting to ZooKeeper ensemble=127.0.0.1:62825 2024-12-03T23:14:33,289 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T23:14:33,291 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T23:14:33,305 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:411670x0, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-03T23:14:33,305 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:411670x0, quorum=127.0.0.1:62825, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-03T23:14:33,306 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-03T23:14:33,317 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-03T23:14:33,319 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:411670x0, quorum=127.0.0.1:62825, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-03T23:14:33,322 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:411670x0, quorum=127.0.0.1:62825, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-03T23:14:33,325 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:41167-0x1017a9bb4a60003 connected 2024-12-03T23:14:33,329 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41167 2024-12-03T23:14:33,329 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41167 2024-12-03T23:14:33,333 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41167 2024-12-03T23:14:33,336 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41167 2024-12-03T23:14:33,336 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41167 2024-12-03T23:14:33,357 DEBUG [M:0;5f9d64419733:44117 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;5f9d64419733:44117 2024-12-03T23:14:33,362 INFO [master/5f9d64419733:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/5f9d64419733,44117,1733267672173 2024-12-03T23:14:33,371 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46833-0x1017a9bb4a60001, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T23:14:33,371 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41167-0x1017a9bb4a60003, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T23:14:33,371 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44899-0x1017a9bb4a60002, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T23:14:33,371 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44117-0x1017a9bb4a60000, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T23:14:33,375 DEBUG [master/5f9d64419733:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44117-0x1017a9bb4a60000, quorum=127.0.0.1:62825, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/5f9d64419733,44117,1733267672173 2024-12-03T23:14:33,404 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46833-0x1017a9bb4a60001, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-03T23:14:33,404 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44899-0x1017a9bb4a60002, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-03T23:14:33,404 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41167-0x1017a9bb4a60003, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-03T23:14:33,405 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46833-0x1017a9bb4a60001, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T23:14:33,405 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44899-0x1017a9bb4a60002, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T23:14:33,405 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44117-0x1017a9bb4a60000, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T23:14:33,405 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41167-0x1017a9bb4a60003, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T23:14:33,407 DEBUG [master/5f9d64419733:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44117-0x1017a9bb4a60000, quorum=127.0.0.1:62825, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-03T23:14:33,410 INFO [master/5f9d64419733:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/5f9d64419733,44117,1733267672173 from backup master directory 2024-12-03T23:14:33,413 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41167-0x1017a9bb4a60003, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T23:14:33,413 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44899-0x1017a9bb4a60002, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T23:14:33,414 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44117-0x1017a9bb4a60000, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/5f9d64419733,44117,1733267672173 2024-12-03T23:14:33,414 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44117-0x1017a9bb4a60000, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T23:14:33,414 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46833-0x1017a9bb4a60001, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T23:14:33,414 WARN [master/5f9d64419733:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-03T23:14:33,415 INFO [master/5f9d64419733:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=5f9d64419733,44117,1733267672173 2024-12-03T23:14:33,420 INFO [master/5f9d64419733:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-03T23:14:33,421 INFO [master/5f9d64419733:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-03T23:14:33,489 DEBUG [master/5f9d64419733:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/hbase.id] with ID: 074196a4-fe11-44a7-af33-90ff39484004 2024-12-03T23:14:33,489 DEBUG [master/5f9d64419733:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/.tmp/hbase.id 2024-12-03T23:14:33,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38883 is added to blk_1073741826_1002 (size=42) 2024-12-03T23:14:33,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44333 is added to blk_1073741826_1002 (size=42) 2024-12-03T23:14:33,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45029 is added to blk_1073741826_1002 (size=42) 2024-12-03T23:14:33,518 DEBUG [master/5f9d64419733:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/.tmp/hbase.id]:[hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/hbase.id] 2024-12-03T23:14:33,586 INFO [master/5f9d64419733:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T23:14:33,593 INFO [master/5f9d64419733:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-03T23:14:33,622 INFO [master/5f9d64419733:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 26ms. 2024-12-03T23:14:33,627 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41167-0x1017a9bb4a60003, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T23:14:33,627 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44899-0x1017a9bb4a60002, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T23:14:33,628 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46833-0x1017a9bb4a60001, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T23:14:33,628 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44117-0x1017a9bb4a60000, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T23:14:33,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44333 is added to blk_1073741827_1003 (size=196) 2024-12-03T23:14:33,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38883 is added to blk_1073741827_1003 (size=196) 2024-12-03T23:14:33,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45029 is added to blk_1073741827_1003 (size=196) 2024-12-03T23:14:33,676 INFO [master/5f9d64419733:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T23:14:33,679 INFO [master/5f9d64419733:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-03T23:14:33,697 DEBUG [master/5f9d64419733:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T23:14:33,702 INFO [master/5f9d64419733:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T23:14:33,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38883 is added to blk_1073741828_1004 (size=1189) 2024-12-03T23:14:33,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44333 is added to blk_1073741828_1004 (size=1189) 2024-12-03T23:14:33,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45029 is added to blk_1073741828_1004 (size=1189) 2024-12-03T23:14:33,764 INFO [master/5f9d64419733:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/MasterData/data/master/store 2024-12-03T23:14:33,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45029 is added to blk_1073741829_1005 (size=34) 2024-12-03T23:14:33,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38883 is added to blk_1073741829_1005 (size=34) 2024-12-03T23:14:33,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44333 is added to blk_1073741829_1005 (size=34) 2024-12-03T23:14:33,807 INFO [master/5f9d64419733:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-03T23:14:33,811 DEBUG [master/5f9d64419733:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T23:14:33,812 DEBUG [master/5f9d64419733:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-03T23:14:33,813 INFO [master/5f9d64419733:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T23:14:33,813 DEBUG [master/5f9d64419733:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T23:14:33,815 DEBUG [master/5f9d64419733:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-03T23:14:33,815 DEBUG [master/5f9d64419733:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T23:14:33,815 INFO [master/5f9d64419733:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T23:14:33,816 DEBUG [master/5f9d64419733:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733267673812Disabling compacts and flushes for region at 1733267673812Disabling writes for close at 1733267673815 (+3 ms)Writing region close event to WAL at 1733267673815Closed at 1733267673815 2024-12-03T23:14:33,819 WARN [master/5f9d64419733:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/MasterData/data/master/store/.initializing 2024-12-03T23:14:33,819 DEBUG [master/5f9d64419733:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/MasterData/WALs/5f9d64419733,44117,1733267672173 2024-12-03T23:14:33,830 INFO [master/5f9d64419733:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T23:14:33,847 INFO [master/5f9d64419733:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=5f9d64419733%2C44117%2C1733267672173, suffix=, logDir=hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/MasterData/WALs/5f9d64419733,44117,1733267672173, archiveDir=hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/MasterData/oldWALs, maxLogs=10 2024-12-03T23:14:33,878 DEBUG [master/5f9d64419733:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/MasterData/WALs/5f9d64419733,44117,1733267672173/5f9d64419733%2C44117%2C1733267672173.1733267673853, exclude list is [], retry=0 2024-12-03T23:14:33,901 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44333,DS-e710ddc2-08f6-4b71-b4db-e94cb5850278,DISK] 2024-12-03T23:14:33,901 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38883,DS-91eef8b1-4e86-469d-90a0-bef7307996a0,DISK] 2024-12-03T23:14:33,901 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45029,DS-e96a58d6-577c-4a7d-bf1d-88064ab38370,DISK] 2024-12-03T23:14:33,905 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-03T23:14:33,953 INFO [master/5f9d64419733:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/MasterData/WALs/5f9d64419733,44117,1733267672173/5f9d64419733%2C44117%2C1733267672173.1733267673853 2024-12-03T23:14:33,954 DEBUG [master/5f9d64419733:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39967:39967),(127.0.0.1/127.0.0.1:34163:34163),(127.0.0.1/127.0.0.1:37111:37111)] 2024-12-03T23:14:33,954 DEBUG [master/5f9d64419733:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-03T23:14:33,955 DEBUG [master/5f9d64419733:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T23:14:33,958 DEBUG [master/5f9d64419733:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T23:14:33,959 DEBUG [master/5f9d64419733:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T23:14:34,013 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-03T23:14:34,047 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-03T23:14:34,051 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:14:34,055 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T23:14:34,055 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-03T23:14:34,059 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-03T23:14:34,060 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:14:34,061 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T23:14:34,062 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-03T23:14:34,066 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-03T23:14:34,066 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:14:34,067 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T23:14:34,068 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-03T23:14:34,071 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-03T23:14:34,071 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:14:34,073 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T23:14:34,074 DEBUG [master/5f9d64419733:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T23:14:34,078 DEBUG [master/5f9d64419733:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-03T23:14:34,079 DEBUG [master/5f9d64419733:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-03T23:14:34,085 DEBUG [master/5f9d64419733:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T23:14:34,085 DEBUG [master/5f9d64419733:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T23:14:34,089 DEBUG [master/5f9d64419733:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-03T23:14:34,093 DEBUG [master/5f9d64419733:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T23:14:34,114 DEBUG [master/5f9d64419733:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T23:14:34,116 INFO [master/5f9d64419733:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64069595, jitterRate=-0.04528863728046417}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-03T23:14:34,126 DEBUG [master/5f9d64419733:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733267673976Initializing all the Stores at 1733267673980 (+4 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733267673981 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733267673982 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733267673983 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733267673983Cleaning up temporary data from old regions at 1733267674085 (+102 ms)Region opened successfully at 1733267674126 (+41 ms) 2024-12-03T23:14:34,128 INFO [master/5f9d64419733:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-03T23:14:34,177 DEBUG [master/5f9d64419733:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64e2b370, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=5f9d64419733/172.17.0.2:0 2024-12-03T23:14:34,218 INFO [master/5f9d64419733:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-03T23:14:34,233 INFO [master/5f9d64419733:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-03T23:14:34,234 INFO [master/5f9d64419733:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-03T23:14:34,237 INFO [master/5f9d64419733:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-03T23:14:34,239 INFO [master/5f9d64419733:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-03T23:14:34,245 INFO [master/5f9d64419733:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-12-03T23:14:34,246 INFO [master/5f9d64419733:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-03T23:14:34,276 INFO [master/5f9d64419733:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-03T23:14:34,287 DEBUG [master/5f9d64419733:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44117-0x1017a9bb4a60000, quorum=127.0.0.1:62825, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-03T23:14:34,291 DEBUG [master/5f9d64419733:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-03T23:14:34,293 INFO [master/5f9d64419733:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-03T23:14:34,295 DEBUG [master/5f9d64419733:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44117-0x1017a9bb4a60000, quorum=127.0.0.1:62825, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-03T23:14:34,297 DEBUG [master/5f9d64419733:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-03T23:14:34,299 INFO [master/5f9d64419733:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-03T23:14:34,303 DEBUG [master/5f9d64419733:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44117-0x1017a9bb4a60000, quorum=127.0.0.1:62825, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-03T23:14:34,305 DEBUG [master/5f9d64419733:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-03T23:14:34,307 DEBUG [master/5f9d64419733:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44117-0x1017a9bb4a60000, quorum=127.0.0.1:62825, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-03T23:14:34,308 DEBUG [master/5f9d64419733:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-03T23:14:34,327 DEBUG [master/5f9d64419733:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44117-0x1017a9bb4a60000, quorum=127.0.0.1:62825, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-03T23:14:34,330 DEBUG [master/5f9d64419733:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-03T23:14:34,334 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41167-0x1017a9bb4a60003, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-03T23:14:34,334 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46833-0x1017a9bb4a60001, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-03T23:14:34,334 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44117-0x1017a9bb4a60000, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-03T23:14:34,334 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41167-0x1017a9bb4a60003, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T23:14:34,335 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44899-0x1017a9bb4a60002, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-03T23:14:34,335 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46833-0x1017a9bb4a60001, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T23:14:34,335 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44899-0x1017a9bb4a60002, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T23:14:34,335 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44117-0x1017a9bb4a60000, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T23:14:34,339 INFO [master/5f9d64419733:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=5f9d64419733,44117,1733267672173, sessionid=0x1017a9bb4a60000, setting cluster-up flag (Was=false) 2024-12-03T23:14:34,352 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46833-0x1017a9bb4a60001, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T23:14:34,352 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44117-0x1017a9bb4a60000, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T23:14:34,352 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41167-0x1017a9bb4a60003, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T23:14:34,352 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44899-0x1017a9bb4a60002, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T23:14:34,360 DEBUG [master/5f9d64419733:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-03T23:14:34,362 DEBUG [master/5f9d64419733:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=5f9d64419733,44117,1733267672173 2024-12-03T23:14:34,369 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41167-0x1017a9bb4a60003, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T23:14:34,369 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46833-0x1017a9bb4a60001, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T23:14:34,369 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44117-0x1017a9bb4a60000, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T23:14:34,370 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44899-0x1017a9bb4a60002, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T23:14:34,376 DEBUG [master/5f9d64419733:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-03T23:14:34,377 DEBUG [master/5f9d64419733:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=5f9d64419733,44117,1733267672173 2024-12-03T23:14:34,384 INFO [master/5f9d64419733:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-03T23:14:34,447 INFO [RS:0;5f9d64419733:46833 {}] regionserver.HRegionServer(746): ClusterId : 074196a4-fe11-44a7-af33-90ff39484004 2024-12-03T23:14:34,450 INFO [RS:2;5f9d64419733:41167 {}] regionserver.HRegionServer(746): ClusterId : 074196a4-fe11-44a7-af33-90ff39484004 2024-12-03T23:14:34,452 DEBUG [RS:2;5f9d64419733:41167 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-03T23:14:34,452 DEBUG [RS:0;5f9d64419733:46833 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-03T23:14:34,462 INFO [RS:1;5f9d64419733:44899 {}] regionserver.HRegionServer(746): ClusterId : 074196a4-fe11-44a7-af33-90ff39484004 2024-12-03T23:14:34,463 DEBUG [RS:1;5f9d64419733:44899 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-03T23:14:34,465 DEBUG [RS:0;5f9d64419733:46833 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-03T23:14:34,465 DEBUG [RS:0;5f9d64419733:46833 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-03T23:14:34,466 DEBUG [RS:1;5f9d64419733:44899 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-03T23:14:34,466 DEBUG [RS:1;5f9d64419733:44899 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-03T23:14:34,470 DEBUG [RS:0;5f9d64419733:46833 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-03T23:14:34,470 DEBUG [RS:0;5f9d64419733:46833 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34dbf841, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=5f9d64419733/172.17.0.2:0 2024-12-03T23:14:34,471 DEBUG [RS:1;5f9d64419733:44899 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-03T23:14:34,471 DEBUG [RS:1;5f9d64419733:44899 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3293af99, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=5f9d64419733/172.17.0.2:0 2024-12-03T23:14:34,477 DEBUG [RS:2;5f9d64419733:41167 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-03T23:14:34,477 DEBUG [RS:2;5f9d64419733:41167 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-03T23:14:34,483 DEBUG [master/5f9d64419733:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-03T23:14:34,487 DEBUG [RS:2;5f9d64419733:41167 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-03T23:14:34,488 DEBUG [RS:2;5f9d64419733:41167 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5effa932, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=5f9d64419733/172.17.0.2:0 2024-12-03T23:14:34,491 DEBUG [RS:1;5f9d64419733:44899 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;5f9d64419733:44899 2024-12-03T23:14:34,494 INFO [RS:1;5f9d64419733:44899 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-03T23:14:34,495 INFO [RS:1;5f9d64419733:44899 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-03T23:14:34,495 DEBUG [RS:1;5f9d64419733:44899 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-03T23:14:34,496 INFO [master/5f9d64419733:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-03T23:14:34,498 INFO [RS:1;5f9d64419733:44899 {}] regionserver.HRegionServer(2659): reportForDuty to master=5f9d64419733,44117,1733267672173 with port=44899, startcode=1733267673202 2024-12-03T23:14:34,504 INFO [master/5f9d64419733:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-03T23:14:34,512 DEBUG [RS:1;5f9d64419733:44899 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-03T23:14:34,514 DEBUG [RS:0;5f9d64419733:46833 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;5f9d64419733:46833 2024-12-03T23:14:34,514 INFO [RS:0;5f9d64419733:46833 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-03T23:14:34,514 INFO [RS:0;5f9d64419733:46833 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-03T23:14:34,514 DEBUG [RS:0;5f9d64419733:46833 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-03T23:14:34,512 DEBUG [master/5f9d64419733:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 5f9d64419733,44117,1733267672173 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-03T23:14:34,516 INFO [RS:0;5f9d64419733:46833 {}] regionserver.HRegionServer(2659): reportForDuty to master=5f9d64419733,44117,1733267672173 with port=46833, startcode=1733267673070 2024-12-03T23:14:34,516 DEBUG [RS:0;5f9d64419733:46833 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-03T23:14:34,521 DEBUG [master/5f9d64419733:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/5f9d64419733:0, corePoolSize=5, maxPoolSize=5 2024-12-03T23:14:34,521 DEBUG [master/5f9d64419733:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/5f9d64419733:0, corePoolSize=5, maxPoolSize=5 2024-12-03T23:14:34,521 DEBUG [master/5f9d64419733:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/5f9d64419733:0, corePoolSize=5, maxPoolSize=5 2024-12-03T23:14:34,522 DEBUG [master/5f9d64419733:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/5f9d64419733:0, corePoolSize=5, maxPoolSize=5 2024-12-03T23:14:34,522 DEBUG [master/5f9d64419733:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/5f9d64419733:0, corePoolSize=10, maxPoolSize=10 2024-12-03T23:14:34,522 DEBUG [master/5f9d64419733:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/5f9d64419733:0, corePoolSize=1, maxPoolSize=1 2024-12-03T23:14:34,522 DEBUG [master/5f9d64419733:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/5f9d64419733:0, corePoolSize=2, maxPoolSize=2 2024-12-03T23:14:34,522 DEBUG [master/5f9d64419733:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/5f9d64419733:0, corePoolSize=1, maxPoolSize=1 2024-12-03T23:14:34,530 DEBUG [RS:2;5f9d64419733:41167 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;5f9d64419733:41167 2024-12-03T23:14:34,530 INFO [RS:2;5f9d64419733:41167 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-03T23:14:34,530 INFO [RS:2;5f9d64419733:41167 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-03T23:14:34,530 DEBUG [RS:2;5f9d64419733:41167 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-03T23:14:34,534 INFO [RS:2;5f9d64419733:41167 {}] regionserver.HRegionServer(2659): reportForDuty to master=5f9d64419733,44117,1733267672173 with port=41167, startcode=1733267673270 2024-12-03T23:14:34,535 DEBUG [RS:2;5f9d64419733:41167 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-03T23:14:34,538 INFO [master/5f9d64419733:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733267704538 2024-12-03T23:14:34,540 INFO [master/5f9d64419733:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-03T23:14:34,542 INFO [master/5f9d64419733:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-03T23:14:34,547 INFO [master/5f9d64419733:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-03T23:14:34,547 INFO [master/5f9d64419733:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-03T23:14:34,548 INFO [master/5f9d64419733:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-03T23:14:34,548 INFO [master/5f9d64419733:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-03T23:14:34,557 INFO [master/5f9d64419733:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-03T23:14:34,559 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-03T23:14:34,561 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-03T23:14:34,562 INFO [master/5f9d64419733:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-03T23:14:34,564 INFO [master/5f9d64419733:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-03T23:14:34,564 INFO [master/5f9d64419733:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-03T23:14:34,578 INFO [master/5f9d64419733:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-03T23:14:34,578 INFO [master/5f9d64419733:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-03T23:14:34,579 INFO [HMasterForTest-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43099, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-12-03T23:14:34,582 INFO [HMasterForTest-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56501, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-12-03T23:14:34,583 INFO [HMasterForTest-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49773, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-03T23:14:34,587 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44117 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T23:14:34,587 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:14:34,588 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-03T23:14:34,589 DEBUG [master/5f9d64419733:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/5f9d64419733:0:becomeActiveMaster-HFileCleaner.large.0-1733267674580,5,FailOnTimeoutGroup] 2024-12-03T23:14:34,596 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44117 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T23:14:34,597 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44117 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T23:14:34,600 DEBUG [master/5f9d64419733:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/5f9d64419733:0:becomeActiveMaster-HFileCleaner.small.0-1733267674589,5,FailOnTimeoutGroup] 2024-12-03T23:14:34,600 INFO [master/5f9d64419733:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-03T23:14:34,601 INFO [master/5f9d64419733:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-03T23:14:34,602 INFO [master/5f9d64419733:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-03T23:14:34,602 INFO [master/5f9d64419733:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-03T23:14:34,623 DEBUG [RS:2;5f9d64419733:41167 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-12-03T23:14:34,623 DEBUG [RS:1;5f9d64419733:44899 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-12-03T23:14:34,623 DEBUG [RS:0;5f9d64419733:46833 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-12-03T23:14:34,623 WARN [RS:1;5f9d64419733:44899 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-12-03T23:14:34,623 WARN [RS:2;5f9d64419733:41167 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-12-03T23:14:34,623 WARN [RS:0;5f9d64419733:46833 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-12-03T23:14:34,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45029 is added to blk_1073741831_1007 (size=1321) 2024-12-03T23:14:34,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38883 is added to blk_1073741831_1007 (size=1321) 2024-12-03T23:14:34,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44333 is added to blk_1073741831_1007 (size=1321) 2024-12-03T23:14:34,656 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-03T23:14:34,656 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1 2024-12-03T23:14:34,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44333 is added to blk_1073741832_1008 (size=32) 2024-12-03T23:14:34,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38883 is added to blk_1073741832_1008 (size=32) 2024-12-03T23:14:34,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45029 is added to blk_1073741832_1008 (size=32) 2024-12-03T23:14:34,685 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T23:14:34,691 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-03T23:14:34,695 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-03T23:14:34,696 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:14:34,699 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T23:14:34,699 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-03T23:14:34,703 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-03T23:14:34,703 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:14:34,704 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T23:14:34,704 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-03T23:14:34,706 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-03T23:14:34,706 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:14:34,708 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T23:14:34,708 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-03T23:14:34,711 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-03T23:14:34,711 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:14:34,712 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T23:14:34,712 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-03T23:14:34,714 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/hbase/meta/1588230740 2024-12-03T23:14:34,715 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/hbase/meta/1588230740 2024-12-03T23:14:34,718 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-03T23:14:34,718 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-03T23:14:34,720 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-03T23:14:34,723 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-03T23:14:34,725 INFO [RS:2;5f9d64419733:41167 {}] regionserver.HRegionServer(2659): reportForDuty to master=5f9d64419733,44117,1733267672173 with port=41167, startcode=1733267673270 2024-12-03T23:14:34,725 INFO [RS:1;5f9d64419733:44899 {}] regionserver.HRegionServer(2659): reportForDuty to master=5f9d64419733,44117,1733267672173 with port=44899, startcode=1733267673202 2024-12-03T23:14:34,725 INFO [RS:0;5f9d64419733:46833 {}] regionserver.HRegionServer(2659): reportForDuty to master=5f9d64419733,44117,1733267672173 with port=46833, startcode=1733267673070 2024-12-03T23:14:34,727 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44117 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 5f9d64419733,46833,1733267673070 2024-12-03T23:14:34,732 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44117 {}] master.ServerManager(517): Registering regionserver=5f9d64419733,46833,1733267673070 2024-12-03T23:14:34,736 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T23:14:34,737 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72920194, jitterRate=0.0865955650806427}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-03T23:14:34,741 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733267674688Initializing all the Stores at 1733267674690 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733267674691 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733267674691Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733267674691Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733267674691Cleaning up temporary data from old regions at 1733267674718 (+27 ms)Region opened successfully at 1733267674741 (+23 ms) 2024-12-03T23:14:34,741 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-03T23:14:34,741 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-03T23:14:34,742 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-03T23:14:34,742 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-03T23:14:34,742 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-03T23:14:34,742 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44117 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 5f9d64419733,41167,1733267673270 2024-12-03T23:14:34,742 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44117 {}] master.ServerManager(517): Registering regionserver=5f9d64419733,41167,1733267673270 2024-12-03T23:14:34,743 DEBUG [RS:0;5f9d64419733:46833 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1 2024-12-03T23:14:34,743 DEBUG [RS:0;5f9d64419733:46833 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:44377 2024-12-03T23:14:34,743 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-03T23:14:34,743 DEBUG [RS:0;5f9d64419733:46833 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-03T23:14:34,743 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733267674741Disabling compacts and flushes for region at 1733267674741Disabling writes for close at 1733267674742 (+1 ms)Writing region close event to WAL at 1733267674743 (+1 ms)Closed at 1733267674743 2024-12-03T23:14:34,748 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44117-0x1017a9bb4a60000, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-03T23:14:34,749 DEBUG [RS:0;5f9d64419733:46833 {}] zookeeper.ZKUtil(111): regionserver:46833-0x1017a9bb4a60001, quorum=127.0.0.1:62825, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/5f9d64419733,46833,1733267673070 2024-12-03T23:14:34,749 WARN [RS:0;5f9d64419733:46833 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-03T23:14:34,750 INFO [RS:0;5f9d64419733:46833 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T23:14:34,750 DEBUG [RS:0;5f9d64419733:46833 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/WALs/5f9d64419733,46833,1733267673070 2024-12-03T23:14:34,753 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [5f9d64419733,46833,1733267673070] 2024-12-03T23:14:34,764 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-03T23:14:34,764 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-03T23:14:34,764 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44117 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 5f9d64419733,44899,1733267673202 2024-12-03T23:14:34,765 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44117 {}] master.ServerManager(517): Registering regionserver=5f9d64419733,44899,1733267673202 2024-12-03T23:14:34,765 DEBUG [RS:2;5f9d64419733:41167 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1 2024-12-03T23:14:34,765 DEBUG [RS:2;5f9d64419733:41167 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:44377 2024-12-03T23:14:34,765 DEBUG [RS:2;5f9d64419733:41167 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-03T23:14:34,771 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44117-0x1017a9bb4a60000, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-03T23:14:34,772 DEBUG [RS:2;5f9d64419733:41167 {}] zookeeper.ZKUtil(111): regionserver:41167-0x1017a9bb4a60003, quorum=127.0.0.1:62825, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/5f9d64419733,41167,1733267673270 2024-12-03T23:14:34,773 WARN [RS:2;5f9d64419733:41167 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-03T23:14:34,773 INFO [RS:2;5f9d64419733:41167 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T23:14:34,773 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [5f9d64419733,41167,1733267673270] 2024-12-03T23:14:34,773 DEBUG [RS:2;5f9d64419733:41167 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/WALs/5f9d64419733,41167,1733267673270 2024-12-03T23:14:34,773 DEBUG [RS:1;5f9d64419733:44899 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1 2024-12-03T23:14:34,773 DEBUG [RS:1;5f9d64419733:44899 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:44377 2024-12-03T23:14:34,773 DEBUG [RS:1;5f9d64419733:44899 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-03T23:14:34,774 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-03T23:14:34,777 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44117-0x1017a9bb4a60000, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-03T23:14:34,778 DEBUG [RS:1;5f9d64419733:44899 {}] zookeeper.ZKUtil(111): regionserver:44899-0x1017a9bb4a60002, quorum=127.0.0.1:62825, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/5f9d64419733,44899,1733267673202 2024-12-03T23:14:34,778 WARN [RS:1;5f9d64419733:44899 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-03T23:14:34,779 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [5f9d64419733,44899,1733267673202] 2024-12-03T23:14:34,779 INFO [RS:1;5f9d64419733:44899 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T23:14:34,779 DEBUG [RS:1;5f9d64419733:44899 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/WALs/5f9d64419733,44899,1733267673202 2024-12-03T23:14:34,798 INFO [RS:0;5f9d64419733:46833 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-03T23:14:34,797 INFO [RS:2;5f9d64419733:41167 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-03T23:14:34,807 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-03T23:14:34,810 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-03T23:14:34,812 INFO [RS:1;5f9d64419733:44899 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-03T23:14:34,823 INFO [RS:0;5f9d64419733:46833 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-03T23:14:34,823 INFO [RS:2;5f9d64419733:41167 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-03T23:14:34,824 INFO [RS:1;5f9d64419733:44899 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-03T23:14:34,829 INFO [RS:1;5f9d64419733:44899 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-03T23:14:34,829 INFO [RS:2;5f9d64419733:41167 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-03T23:14:34,830 INFO [RS:1;5f9d64419733:44899 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T23:14:34,830 INFO [RS:2;5f9d64419733:41167 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T23:14:34,830 INFO [RS:0;5f9d64419733:46833 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-03T23:14:34,831 INFO [RS:2;5f9d64419733:41167 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-03T23:14:34,831 INFO [RS:1;5f9d64419733:44899 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-03T23:14:34,836 INFO [RS:0;5f9d64419733:46833 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T23:14:34,837 INFO [RS:0;5f9d64419733:46833 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-03T23:14:34,843 INFO [RS:2;5f9d64419733:41167 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-03T23:14:34,843 INFO [RS:0;5f9d64419733:46833 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-03T23:14:34,843 INFO [RS:1;5f9d64419733:44899 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-03T23:14:34,845 INFO [RS:2;5f9d64419733:41167 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-03T23:14:34,845 INFO [RS:0;5f9d64419733:46833 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-03T23:14:34,845 INFO [RS:1;5f9d64419733:44899 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-03T23:14:34,845 DEBUG [RS:1;5f9d64419733:44899 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/5f9d64419733:0, corePoolSize=1, maxPoolSize=1 2024-12-03T23:14:34,845 DEBUG [RS:0;5f9d64419733:46833 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/5f9d64419733:0, corePoolSize=1, maxPoolSize=1 2024-12-03T23:14:34,845 DEBUG [RS:2;5f9d64419733:41167 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/5f9d64419733:0, corePoolSize=1, maxPoolSize=1 2024-12-03T23:14:34,845 DEBUG [RS:1;5f9d64419733:44899 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/5f9d64419733:0, corePoolSize=1, maxPoolSize=1 2024-12-03T23:14:34,845 DEBUG [RS:2;5f9d64419733:41167 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/5f9d64419733:0, corePoolSize=1, maxPoolSize=1 2024-12-03T23:14:34,845 DEBUG [RS:0;5f9d64419733:46833 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/5f9d64419733:0, corePoolSize=1, maxPoolSize=1 2024-12-03T23:14:34,846 DEBUG [RS:1;5f9d64419733:44899 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/5f9d64419733:0, corePoolSize=1, maxPoolSize=1 2024-12-03T23:14:34,846 DEBUG [RS:2;5f9d64419733:41167 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/5f9d64419733:0, corePoolSize=1, maxPoolSize=1 2024-12-03T23:14:34,846 DEBUG [RS:1;5f9d64419733:44899 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/5f9d64419733:0, corePoolSize=1, maxPoolSize=1 2024-12-03T23:14:34,846 DEBUG [RS:2;5f9d64419733:41167 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/5f9d64419733:0, corePoolSize=1, maxPoolSize=1 2024-12-03T23:14:34,846 DEBUG [RS:1;5f9d64419733:44899 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/5f9d64419733:0, corePoolSize=1, maxPoolSize=1 2024-12-03T23:14:34,846 DEBUG [RS:2;5f9d64419733:41167 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/5f9d64419733:0, corePoolSize=1, maxPoolSize=1 2024-12-03T23:14:34,846 DEBUG [RS:1;5f9d64419733:44899 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/5f9d64419733:0, corePoolSize=2, maxPoolSize=2 2024-12-03T23:14:34,846 DEBUG [RS:2;5f9d64419733:41167 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/5f9d64419733:0, corePoolSize=2, maxPoolSize=2 2024-12-03T23:14:34,846 DEBUG [RS:1;5f9d64419733:44899 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/5f9d64419733:0, corePoolSize=1, maxPoolSize=1 2024-12-03T23:14:34,846 DEBUG [RS:2;5f9d64419733:41167 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/5f9d64419733:0, corePoolSize=1, maxPoolSize=1 2024-12-03T23:14:34,846 DEBUG [RS:2;5f9d64419733:41167 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/5f9d64419733:0, corePoolSize=1, maxPoolSize=1 2024-12-03T23:14:34,846 DEBUG [RS:1;5f9d64419733:44899 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/5f9d64419733:0, corePoolSize=1, maxPoolSize=1 2024-12-03T23:14:34,847 DEBUG [RS:2;5f9d64419733:41167 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/5f9d64419733:0, corePoolSize=1, maxPoolSize=1 2024-12-03T23:14:34,847 DEBUG [RS:1;5f9d64419733:44899 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/5f9d64419733:0, corePoolSize=1, maxPoolSize=1 2024-12-03T23:14:34,847 DEBUG [RS:2;5f9d64419733:41167 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/5f9d64419733:0, corePoolSize=1, maxPoolSize=1 2024-12-03T23:14:34,847 DEBUG [RS:1;5f9d64419733:44899 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/5f9d64419733:0, corePoolSize=1, maxPoolSize=1 2024-12-03T23:14:34,847 DEBUG [RS:1;5f9d64419733:44899 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/5f9d64419733:0, corePoolSize=1, maxPoolSize=1 2024-12-03T23:14:34,847 DEBUG [RS:2;5f9d64419733:41167 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/5f9d64419733:0, corePoolSize=1, maxPoolSize=1 2024-12-03T23:14:34,847 DEBUG [RS:1;5f9d64419733:44899 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/5f9d64419733:0, corePoolSize=1, maxPoolSize=1 2024-12-03T23:14:34,847 DEBUG [RS:2;5f9d64419733:41167 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/5f9d64419733:0, corePoolSize=1, maxPoolSize=1 2024-12-03T23:14:34,847 DEBUG [RS:1;5f9d64419733:44899 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/5f9d64419733:0, corePoolSize=3, maxPoolSize=3 2024-12-03T23:14:34,847 DEBUG [RS:2;5f9d64419733:41167 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/5f9d64419733:0, corePoolSize=3, maxPoolSize=3 2024-12-03T23:14:34,847 DEBUG [RS:1;5f9d64419733:44899 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/5f9d64419733:0, corePoolSize=3, maxPoolSize=3 2024-12-03T23:14:34,847 DEBUG [RS:2;5f9d64419733:41167 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/5f9d64419733:0, corePoolSize=3, maxPoolSize=3 2024-12-03T23:14:34,848 DEBUG [RS:0;5f9d64419733:46833 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/5f9d64419733:0, corePoolSize=1, maxPoolSize=1 2024-12-03T23:14:34,848 DEBUG [RS:0;5f9d64419733:46833 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/5f9d64419733:0, corePoolSize=1, maxPoolSize=1 2024-12-03T23:14:34,848 DEBUG [RS:0;5f9d64419733:46833 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/5f9d64419733:0, corePoolSize=1, maxPoolSize=1 2024-12-03T23:14:34,848 DEBUG [RS:0;5f9d64419733:46833 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/5f9d64419733:0, corePoolSize=2, maxPoolSize=2 2024-12-03T23:14:34,848 DEBUG [RS:0;5f9d64419733:46833 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/5f9d64419733:0, corePoolSize=1, maxPoolSize=1 2024-12-03T23:14:34,849 DEBUG [RS:0;5f9d64419733:46833 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/5f9d64419733:0, corePoolSize=1, maxPoolSize=1 2024-12-03T23:14:34,849 DEBUG [RS:0;5f9d64419733:46833 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/5f9d64419733:0, corePoolSize=1, maxPoolSize=1 2024-12-03T23:14:34,849 DEBUG [RS:0;5f9d64419733:46833 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/5f9d64419733:0, corePoolSize=1, maxPoolSize=1 2024-12-03T23:14:34,849 DEBUG [RS:0;5f9d64419733:46833 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/5f9d64419733:0, corePoolSize=1, maxPoolSize=1 2024-12-03T23:14:34,849 DEBUG [RS:0;5f9d64419733:46833 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/5f9d64419733:0, corePoolSize=1, maxPoolSize=1 2024-12-03T23:14:34,850 DEBUG [RS:0;5f9d64419733:46833 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/5f9d64419733:0, corePoolSize=3, maxPoolSize=3 2024-12-03T23:14:34,850 DEBUG [RS:0;5f9d64419733:46833 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/5f9d64419733:0, corePoolSize=3, maxPoolSize=3 2024-12-03T23:14:34,856 INFO [RS:1;5f9d64419733:44899 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-03T23:14:34,857 INFO [RS:1;5f9d64419733:44899 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-03T23:14:34,857 INFO [RS:2;5f9d64419733:41167 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-03T23:14:34,857 INFO [RS:1;5f9d64419733:44899 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T23:14:34,857 INFO [RS:2;5f9d64419733:41167 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-03T23:14:34,857 INFO [RS:1;5f9d64419733:44899 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-03T23:14:34,857 INFO [RS:2;5f9d64419733:41167 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T23:14:34,857 INFO [RS:1;5f9d64419733:44899 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-03T23:14:34,857 INFO [RS:2;5f9d64419733:41167 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-03T23:14:34,857 INFO [RS:1;5f9d64419733:44899 {}] hbase.ChoreService(168): Chore ScheduledChore name=5f9d64419733,44899,1733267673202-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-03T23:14:34,857 INFO [RS:2;5f9d64419733:41167 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-03T23:14:34,857 INFO [RS:2;5f9d64419733:41167 {}] hbase.ChoreService(168): Chore ScheduledChore name=5f9d64419733,41167,1733267673270-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-03T23:14:34,860 INFO [RS:0;5f9d64419733:46833 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-03T23:14:34,860 INFO [RS:0;5f9d64419733:46833 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-03T23:14:34,861 INFO [RS:0;5f9d64419733:46833 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T23:14:34,861 INFO [RS:0;5f9d64419733:46833 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-03T23:14:34,861 INFO [RS:0;5f9d64419733:46833 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-03T23:14:34,861 INFO [RS:0;5f9d64419733:46833 {}] hbase.ChoreService(168): Chore ScheduledChore name=5f9d64419733,46833,1733267673070-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-03T23:14:34,889 INFO [RS:1;5f9d64419733:44899 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-03T23:14:34,891 INFO [RS:1;5f9d64419733:44899 {}] hbase.ChoreService(168): Chore ScheduledChore name=5f9d64419733,44899,1733267673202-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T23:14:34,892 INFO [RS:1;5f9d64419733:44899 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T23:14:34,892 INFO [RS:1;5f9d64419733:44899 {}] regionserver.Replication(171): 5f9d64419733,44899,1733267673202 started 2024-12-03T23:14:34,896 INFO [RS:0;5f9d64419733:46833 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-03T23:14:34,896 INFO [RS:0;5f9d64419733:46833 {}] hbase.ChoreService(168): Chore ScheduledChore name=5f9d64419733,46833,1733267673070-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T23:14:34,897 INFO [RS:0;5f9d64419733:46833 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T23:14:34,897 INFO [RS:0;5f9d64419733:46833 {}] regionserver.Replication(171): 5f9d64419733,46833,1733267673070 started 2024-12-03T23:14:34,904 INFO [RS:2;5f9d64419733:41167 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-03T23:14:34,904 INFO [RS:2;5f9d64419733:41167 {}] hbase.ChoreService(168): Chore ScheduledChore name=5f9d64419733,41167,1733267673270-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T23:14:34,904 INFO [RS:2;5f9d64419733:41167 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T23:14:34,904 INFO [RS:2;5f9d64419733:41167 {}] regionserver.Replication(171): 5f9d64419733,41167,1733267673270 started 2024-12-03T23:14:34,920 INFO [RS:0;5f9d64419733:46833 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T23:14:34,920 INFO [RS:1;5f9d64419733:44899 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T23:14:34,920 INFO [RS:0;5f9d64419733:46833 {}] regionserver.HRegionServer(1482): Serving as 5f9d64419733,46833,1733267673070, RpcServer on 5f9d64419733/172.17.0.2:46833, sessionid=0x1017a9bb4a60001 2024-12-03T23:14:34,920 INFO [RS:1;5f9d64419733:44899 {}] regionserver.HRegionServer(1482): Serving as 5f9d64419733,44899,1733267673202, RpcServer on 5f9d64419733/172.17.0.2:44899, sessionid=0x1017a9bb4a60002 2024-12-03T23:14:34,921 DEBUG [RS:1;5f9d64419733:44899 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-03T23:14:34,921 DEBUG [RS:0;5f9d64419733:46833 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-03T23:14:34,922 DEBUG [RS:1;5f9d64419733:44899 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 5f9d64419733,44899,1733267673202 2024-12-03T23:14:34,922 DEBUG [RS:0;5f9d64419733:46833 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 5f9d64419733,46833,1733267673070 2024-12-03T23:14:34,922 DEBUG [RS:0;5f9d64419733:46833 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '5f9d64419733,46833,1733267673070' 2024-12-03T23:14:34,922 DEBUG [RS:1;5f9d64419733:44899 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '5f9d64419733,44899,1733267673202' 2024-12-03T23:14:34,922 DEBUG [RS:0;5f9d64419733:46833 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-03T23:14:34,922 DEBUG [RS:1;5f9d64419733:44899 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-03T23:14:34,923 DEBUG [RS:0;5f9d64419733:46833 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-03T23:14:34,924 DEBUG [RS:0;5f9d64419733:46833 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-03T23:14:34,924 DEBUG [RS:0;5f9d64419733:46833 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-03T23:14:34,925 DEBUG [RS:0;5f9d64419733:46833 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 5f9d64419733,46833,1733267673070 2024-12-03T23:14:34,925 DEBUG [RS:0;5f9d64419733:46833 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '5f9d64419733,46833,1733267673070' 2024-12-03T23:14:34,925 DEBUG [RS:0;5f9d64419733:46833 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-03T23:14:34,926 DEBUG [RS:0;5f9d64419733:46833 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-03T23:14:34,926 DEBUG [RS:1;5f9d64419733:44899 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-03T23:14:34,926 DEBUG [RS:0;5f9d64419733:46833 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-03T23:14:34,927 INFO [RS:0;5f9d64419733:46833 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-03T23:14:34,927 DEBUG [RS:1;5f9d64419733:44899 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-03T23:14:34,927 INFO [RS:0;5f9d64419733:46833 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-03T23:14:34,927 DEBUG [RS:1;5f9d64419733:44899 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-03T23:14:34,927 DEBUG [RS:1;5f9d64419733:44899 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 5f9d64419733,44899,1733267673202 2024-12-03T23:14:34,927 DEBUG [RS:1;5f9d64419733:44899 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '5f9d64419733,44899,1733267673202' 2024-12-03T23:14:34,927 DEBUG [RS:1;5f9d64419733:44899 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-03T23:14:34,928 DEBUG [RS:1;5f9d64419733:44899 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-03T23:14:34,931 DEBUG [RS:1;5f9d64419733:44899 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-03T23:14:34,931 INFO [RS:1;5f9d64419733:44899 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-03T23:14:34,931 INFO [RS:1;5f9d64419733:44899 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-03T23:14:34,938 INFO [RS:2;5f9d64419733:41167 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T23:14:34,938 INFO [RS:2;5f9d64419733:41167 {}] regionserver.HRegionServer(1482): Serving as 5f9d64419733,41167,1733267673270, RpcServer on 5f9d64419733/172.17.0.2:41167, sessionid=0x1017a9bb4a60003 2024-12-03T23:14:34,938 DEBUG [RS:2;5f9d64419733:41167 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-03T23:14:34,938 DEBUG [RS:2;5f9d64419733:41167 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 5f9d64419733,41167,1733267673270 2024-12-03T23:14:34,938 DEBUG [RS:2;5f9d64419733:41167 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '5f9d64419733,41167,1733267673270' 2024-12-03T23:14:34,938 DEBUG [RS:2;5f9d64419733:41167 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-03T23:14:34,939 DEBUG [RS:2;5f9d64419733:41167 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-03T23:14:34,942 DEBUG [RS:2;5f9d64419733:41167 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-03T23:14:34,942 DEBUG [RS:2;5f9d64419733:41167 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-03T23:14:34,942 DEBUG [RS:2;5f9d64419733:41167 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 5f9d64419733,41167,1733267673270 2024-12-03T23:14:34,942 DEBUG [RS:2;5f9d64419733:41167 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '5f9d64419733,41167,1733267673270' 2024-12-03T23:14:34,942 DEBUG [RS:2;5f9d64419733:41167 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-03T23:14:34,943 DEBUG [RS:2;5f9d64419733:41167 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-03T23:14:34,944 DEBUG [RS:2;5f9d64419733:41167 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-03T23:14:34,944 INFO [RS:2;5f9d64419733:41167 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-03T23:14:34,944 INFO [RS:2;5f9d64419733:41167 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-03T23:14:34,961 WARN [5f9d64419733:44117 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T23:14:35,033 INFO [RS:0;5f9d64419733:46833 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T23:14:35,033 INFO [RS:1;5f9d64419733:44899 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T23:14:35,036 INFO [RS:1;5f9d64419733:44899 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=5f9d64419733%2C44899%2C1733267673202, suffix=, logDir=hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/WALs/5f9d64419733,44899,1733267673202, archiveDir=hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/oldWALs, maxLogs=32 2024-12-03T23:14:35,036 INFO [RS:0;5f9d64419733:46833 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=5f9d64419733%2C46833%2C1733267673070, suffix=, logDir=hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/WALs/5f9d64419733,46833,1733267673070, archiveDir=hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/oldWALs, maxLogs=32 2024-12-03T23:14:35,045 INFO [RS:2;5f9d64419733:41167 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T23:14:35,048 INFO [RS:2;5f9d64419733:41167 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=5f9d64419733%2C41167%2C1733267673270, suffix=, logDir=hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/WALs/5f9d64419733,41167,1733267673270, archiveDir=hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/oldWALs, maxLogs=32 2024-12-03T23:14:35,063 DEBUG [RS:0;5f9d64419733:46833 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/WALs/5f9d64419733,46833,1733267673070/5f9d64419733%2C46833%2C1733267673070.1733267675041, exclude list is [], retry=0 2024-12-03T23:14:35,064 DEBUG [RS:1;5f9d64419733:44899 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/WALs/5f9d64419733,44899,1733267673202/5f9d64419733%2C44899%2C1733267673202.1733267675041, exclude list is [], retry=0 2024-12-03T23:14:35,066 DEBUG [RS:2;5f9d64419733:41167 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/WALs/5f9d64419733,41167,1733267673270/5f9d64419733%2C41167%2C1733267673270.1733267675050, exclude list is [], retry=0 2024-12-03T23:14:35,071 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45029,DS-e96a58d6-577c-4a7d-bf1d-88064ab38370,DISK] 2024-12-03T23:14:35,071 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44333,DS-e710ddc2-08f6-4b71-b4db-e94cb5850278,DISK] 2024-12-03T23:14:35,072 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38883,DS-91eef8b1-4e86-469d-90a0-bef7307996a0,DISK] 2024-12-03T23:14:35,073 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44333,DS-e710ddc2-08f6-4b71-b4db-e94cb5850278,DISK] 2024-12-03T23:14:35,073 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38883,DS-91eef8b1-4e86-469d-90a0-bef7307996a0,DISK] 2024-12-03T23:14:35,074 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45029,DS-e96a58d6-577c-4a7d-bf1d-88064ab38370,DISK] 2024-12-03T23:14:35,074 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45029,DS-e96a58d6-577c-4a7d-bf1d-88064ab38370,DISK] 2024-12-03T23:14:35,074 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38883,DS-91eef8b1-4e86-469d-90a0-bef7307996a0,DISK] 2024-12-03T23:14:35,074 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44333,DS-e710ddc2-08f6-4b71-b4db-e94cb5850278,DISK] 2024-12-03T23:14:35,091 INFO [RS:0;5f9d64419733:46833 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/WALs/5f9d64419733,46833,1733267673070/5f9d64419733%2C46833%2C1733267673070.1733267675041 2024-12-03T23:14:35,095 INFO [RS:1;5f9d64419733:44899 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/WALs/5f9d64419733,44899,1733267673202/5f9d64419733%2C44899%2C1733267673202.1733267675041 2024-12-03T23:14:35,099 DEBUG [RS:0;5f9d64419733:46833 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39967:39967),(127.0.0.1/127.0.0.1:34163:34163),(127.0.0.1/127.0.0.1:37111:37111)] 2024-12-03T23:14:35,099 DEBUG [RS:1;5f9d64419733:44899 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39967:39967),(127.0.0.1/127.0.0.1:34163:34163),(127.0.0.1/127.0.0.1:37111:37111)] 2024-12-03T23:14:35,104 INFO [RS:2;5f9d64419733:41167 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/WALs/5f9d64419733,41167,1733267673270/5f9d64419733%2C41167%2C1733267673270.1733267675050 2024-12-03T23:14:35,105 DEBUG [RS:2;5f9d64419733:41167 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34163:34163),(127.0.0.1/127.0.0.1:37111:37111),(127.0.0.1/127.0.0.1:39967:39967)] 2024-12-03T23:14:35,214 DEBUG [5f9d64419733:44117 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-12-03T23:14:35,225 DEBUG [5f9d64419733:44117 {}] balancer.BalancerClusterState(204): Hosts are {5f9d64419733=0} racks are {/default-rack=0} 2024-12-03T23:14:35,234 DEBUG [5f9d64419733:44117 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-03T23:14:35,234 DEBUG [5f9d64419733:44117 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-03T23:14:35,234 DEBUG [5f9d64419733:44117 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-12-03T23:14:35,234 DEBUG [5f9d64419733:44117 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-03T23:14:35,234 DEBUG [5f9d64419733:44117 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-03T23:14:35,234 DEBUG [5f9d64419733:44117 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-12-03T23:14:35,234 INFO [5f9d64419733:44117 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-03T23:14:35,234 INFO [5f9d64419733:44117 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-03T23:14:35,234 INFO [5f9d64419733:44117 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-12-03T23:14:35,234 DEBUG [5f9d64419733:44117 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-03T23:14:35,243 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=5f9d64419733,46833,1733267673070 2024-12-03T23:14:35,250 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 5f9d64419733,46833,1733267673070, state=OPENING 2024-12-03T23:14:35,256 DEBUG [PEWorker-1 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-03T23:14:35,258 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41167-0x1017a9bb4a60003, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T23:14:35,258 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46833-0x1017a9bb4a60001, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T23:14:35,258 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44117-0x1017a9bb4a60000, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T23:14:35,258 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44899-0x1017a9bb4a60002, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T23:14:35,259 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T23:14:35,259 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T23:14:35,259 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T23:14:35,259 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T23:14:35,261 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-03T23:14:35,263 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=5f9d64419733,46833,1733267673070}] 2024-12-03T23:14:35,445 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-03T23:14:35,448 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42723, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-03T23:14:35,463 INFO [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-03T23:14:35,463 INFO [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T23:14:35,463 INFO [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-03T23:14:35,467 INFO [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=5f9d64419733%2C46833%2C1733267673070.meta, suffix=.meta, logDir=hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/WALs/5f9d64419733,46833,1733267673070, archiveDir=hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/oldWALs, maxLogs=32 2024-12-03T23:14:35,486 DEBUG [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/WALs/5f9d64419733,46833,1733267673070/5f9d64419733%2C46833%2C1733267673070.meta.1733267675470.meta, exclude list is [], retry=0 2024-12-03T23:14:35,490 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38883,DS-91eef8b1-4e86-469d-90a0-bef7307996a0,DISK] 2024-12-03T23:14:35,490 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45029,DS-e96a58d6-577c-4a7d-bf1d-88064ab38370,DISK] 2024-12-03T23:14:35,491 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44333,DS-e710ddc2-08f6-4b71-b4db-e94cb5850278,DISK] 2024-12-03T23:14:35,500 INFO [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/WALs/5f9d64419733,46833,1733267673070/5f9d64419733%2C46833%2C1733267673070.meta.1733267675470.meta 2024-12-03T23:14:35,501 DEBUG [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37111:37111),(127.0.0.1/127.0.0.1:34163:34163),(127.0.0.1/127.0.0.1:39967:39967)] 2024-12-03T23:14:35,501 DEBUG [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-03T23:14:35,503 DEBUG [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-03T23:14:35,506 DEBUG [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-03T23:14:35,513 INFO [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-03T23:14:35,518 DEBUG [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-03T23:14:35,519 DEBUG [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T23:14:35,519 DEBUG [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-03T23:14:35,519 DEBUG [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-03T23:14:35,524 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-03T23:14:35,525 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-03T23:14:35,526 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:14:35,526 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T23:14:35,527 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-03T23:14:35,528 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-03T23:14:35,528 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:14:35,529 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T23:14:35,529 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-03T23:14:35,531 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-03T23:14:35,531 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:14:35,531 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T23:14:35,532 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-03T23:14:35,533 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-03T23:14:35,533 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:14:35,534 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T23:14:35,534 DEBUG [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-03T23:14:35,536 DEBUG [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/hbase/meta/1588230740 2024-12-03T23:14:35,540 DEBUG [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/hbase/meta/1588230740 2024-12-03T23:14:35,550 DEBUG [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-03T23:14:35,550 DEBUG [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-03T23:14:35,552 DEBUG [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-03T23:14:35,554 DEBUG [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-03T23:14:35,558 INFO [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62890854, jitterRate=-0.06285324692726135}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-03T23:14:35,558 DEBUG [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-03T23:14:35,560 DEBUG [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733267675520Writing region info on filesystem at 1733267675520Initializing all the Stores at 1733267675523 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733267675523Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733267675523Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733267675523Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733267675523Cleaning up temporary data from old regions at 1733267675550 (+27 ms)Running coprocessor post-open hooks at 1733267675558 (+8 ms)Region opened successfully at 1733267675560 (+2 ms) 2024-12-03T23:14:35,577 INFO [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733267675433 2024-12-03T23:14:35,596 DEBUG [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-03T23:14:35,597 INFO [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-03T23:14:35,601 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=5f9d64419733,46833,1733267673070 2024-12-03T23:14:35,605 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 5f9d64419733,46833,1733267673070, state=OPEN 2024-12-03T23:14:35,609 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44899-0x1017a9bb4a60002, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T23:14:35,610 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46833-0x1017a9bb4a60001, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T23:14:35,610 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41167-0x1017a9bb4a60003, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T23:14:35,610 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T23:14:35,610 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T23:14:35,610 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T23:14:35,610 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44117-0x1017a9bb4a60000, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T23:14:35,611 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T23:14:35,614 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=5f9d64419733,46833,1733267673070 2024-12-03T23:14:35,625 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-03T23:14:35,625 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=5f9d64419733,46833,1733267673070 in 351 msec 2024-12-03T23:14:35,637 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-03T23:14:35,638 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 855 msec 2024-12-03T23:14:35,640 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-03T23:14:35,640 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-03T23:14:35,687 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T23:14:35,689 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5f9d64419733,46833,1733267673070, seqNum=-1] 2024-12-03T23:14:35,722 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T23:14:35,730 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47769, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T23:14:35,769 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.3400 sec 2024-12-03T23:14:35,770 INFO [master/5f9d64419733:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733267675770, completionTime=-1 2024-12-03T23:14:35,773 INFO [master/5f9d64419733:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-12-03T23:14:35,773 DEBUG [master/5f9d64419733:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-03T23:14:35,812 INFO [master/5f9d64419733:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=3 2024-12-03T23:14:35,812 INFO [master/5f9d64419733:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733267735812 2024-12-03T23:14:35,812 INFO [master/5f9d64419733:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733267795812 2024-12-03T23:14:35,812 INFO [master/5f9d64419733:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 38 msec 2024-12-03T23:14:35,814 DEBUG [master/5f9d64419733:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-12-03T23:14:35,834 INFO [master/5f9d64419733:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=5f9d64419733,44117,1733267672173-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T23:14:35,834 INFO [master/5f9d64419733:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=5f9d64419733,44117,1733267672173-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T23:14:35,835 INFO [master/5f9d64419733:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=5f9d64419733,44117,1733267672173-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T23:14:35,837 INFO [master/5f9d64419733:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-5f9d64419733:44117, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T23:14:35,847 DEBUG [master/5f9d64419733:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-03T23:14:35,848 INFO [master/5f9d64419733:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-03T23:14:35,852 INFO [master/5f9d64419733:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-03T23:14:35,897 INFO [master/5f9d64419733:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.482sec 2024-12-03T23:14:35,900 INFO [master/5f9d64419733:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-03T23:14:35,902 INFO [master/5f9d64419733:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-03T23:14:35,903 INFO [master/5f9d64419733:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-03T23:14:35,904 INFO [master/5f9d64419733:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-03T23:14:35,904 INFO [master/5f9d64419733:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-03T23:14:35,905 INFO [master/5f9d64419733:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=5f9d64419733,44117,1733267672173-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-03T23:14:35,905 INFO [master/5f9d64419733:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=5f9d64419733,44117,1733267672173-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-03T23:14:35,921 DEBUG [master/5f9d64419733:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-03T23:14:35,922 INFO [master/5f9d64419733:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-03T23:14:35,923 INFO [master/5f9d64419733:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=5f9d64419733,44117,1733267672173-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T23:14:35,966 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@117ba4ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T23:14:35,972 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-03T23:14:35,972 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-03T23:14:35,978 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5f9d64419733,44117,-1 for getting cluster id 2024-12-03T23:14:35,982 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T23:14:35,995 DEBUG [HMasterForTest-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '074196a4-fe11-44a7-af33-90ff39484004' 2024-12-03T23:14:35,998 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T23:14:35,998 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "074196a4-fe11-44a7-af33-90ff39484004" 2024-12-03T23:14:35,999 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70c22f79, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T23:14:35,999 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5f9d64419733,44117,-1] 2024-12-03T23:14:36,002 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T23:14:36,020 INFO [HMasterForTest-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54126, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T23:14:36,020 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T23:14:36,023 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2fc9d8e7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T23:14:36,024 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T23:14:36,037 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5f9d64419733,46833,1733267673070, seqNum=-1] 2024-12-03T23:14:36,038 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T23:14:36,042 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35220, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T23:14:36,069 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=5f9d64419733,44117,1733267672173 2024-12-03T23:14:36,074 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-03T23:14:36,079 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 5f9d64419733,44117,1733267672173 2024-12-03T23:14:36,082 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7cb19ea8 2024-12-03T23:14:36,083 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-03T23:14:36,088 INFO [HMasterForTest-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54130, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-03T23:14:36,096 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44117 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'test', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T23:14:36,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44117 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=test 2024-12-03T23:14:36,141 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_PRE_OPERATION 2024-12-03T23:14:36,146 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:14:36,153 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-03T23:14:36,155 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44117 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "test" procId is: 4 2024-12-03T23:14:36,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-03T23:14:36,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38883 is added to blk_1073741837_1013 (size=358) 2024-12-03T23:14:36,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45029 is added to blk_1073741837_1013 (size=358) 2024-12-03T23:14:36,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44333 is added to blk_1073741837_1013 (size=358) 2024-12-03T23:14:36,198 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 6d9981bd2d541debb970a37ac121e7ed, NAME => 'test,,1733267676090.6d9981bd2d541debb970a37ac121e7ed.', STARTKEY => '', ENDKEY => 'bbb'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1 2024-12-03T23:14:36,198 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 0e35e57b6f6cbff01d3e782fbf6efd1f, NAME => 'test,ccc,1733267676090.0e35e57b6f6cbff01d3e782fbf6efd1f.', STARTKEY => 'ccc', ENDKEY => 'ddd'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1 2024-12-03T23:14:36,198 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => f71f90ef2d2f59e885f2ad6ca1a99596, NAME => 'test,bbb,1733267676090.f71f90ef2d2f59e885f2ad6ca1a99596.', STARTKEY => 'bbb', ENDKEY => 'ccc'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1 2024-12-03T23:14:36,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45029 is added to blk_1073741838_1014 (size=45) 2024-12-03T23:14:36,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44333 is added to blk_1073741838_1014 (size=45) 2024-12-03T23:14:36,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38883 is added to blk_1073741838_1014 (size=45) 2024-12-03T23:14:36,226 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ccc,1733267676090.0e35e57b6f6cbff01d3e782fbf6efd1f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T23:14:36,227 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 0e35e57b6f6cbff01d3e782fbf6efd1f, disabling compactions & flushes 2024-12-03T23:14:36,227 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ccc,1733267676090.0e35e57b6f6cbff01d3e782fbf6efd1f. 2024-12-03T23:14:36,227 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ccc,1733267676090.0e35e57b6f6cbff01d3e782fbf6efd1f. 2024-12-03T23:14:36,227 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ccc,1733267676090.0e35e57b6f6cbff01d3e782fbf6efd1f. after waiting 0 ms 2024-12-03T23:14:36,227 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ccc,1733267676090.0e35e57b6f6cbff01d3e782fbf6efd1f. 2024-12-03T23:14:36,227 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ccc,1733267676090.0e35e57b6f6cbff01d3e782fbf6efd1f. 2024-12-03T23:14:36,227 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 0e35e57b6f6cbff01d3e782fbf6efd1f: Waiting for close lock at 1733267676227Disabling compacts and flushes for region at 1733267676227Disabling writes for close at 1733267676227Writing region close event to WAL at 1733267676227Closed at 1733267676227 2024-12-03T23:14:36,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44333 is added to blk_1073741839_1015 (size=45) 2024-12-03T23:14:36,228 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 55eff67791415c655cef877182b9bbe1, NAME => 'test,ddd,1733267676090.55eff67791415c655cef877182b9bbe1.', STARTKEY => 'ddd', ENDKEY => 'eee'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1 2024-12-03T23:14:36,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38883 is added to blk_1073741839_1015 (size=45) 2024-12-03T23:14:36,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44333 is added to blk_1073741840_1016 (size=42) 2024-12-03T23:14:36,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45029 is added to blk_1073741840_1016 (size=42) 2024-12-03T23:14:36,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38883 is added to blk_1073741840_1016 (size=42) 2024-12-03T23:14:36,231 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,bbb,1733267676090.f71f90ef2d2f59e885f2ad6ca1a99596.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T23:14:36,231 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing f71f90ef2d2f59e885f2ad6ca1a99596, disabling compactions & flushes 2024-12-03T23:14:36,231 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,bbb,1733267676090.f71f90ef2d2f59e885f2ad6ca1a99596. 2024-12-03T23:14:36,231 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,bbb,1733267676090.f71f90ef2d2f59e885f2ad6ca1a99596. 2024-12-03T23:14:36,231 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,bbb,1733267676090.f71f90ef2d2f59e885f2ad6ca1a99596. after waiting 0 ms 2024-12-03T23:14:36,231 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,bbb,1733267676090.f71f90ef2d2f59e885f2ad6ca1a99596. 2024-12-03T23:14:36,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45029 is added to blk_1073741839_1015 (size=45) 2024-12-03T23:14:36,231 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,bbb,1733267676090.f71f90ef2d2f59e885f2ad6ca1a99596. 2024-12-03T23:14:36,232 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for f71f90ef2d2f59e885f2ad6ca1a99596: Waiting for close lock at 1733267676231Disabling compacts and flushes for region at 1733267676231Disabling writes for close at 1733267676231Writing region close event to WAL at 1733267676231Closed at 1733267676231 2024-12-03T23:14:36,232 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 212a9963c9cddede26a4eb521cff5254, NAME => 'test,eee,1733267676090.212a9963c9cddede26a4eb521cff5254.', STARTKEY => 'eee', ENDKEY => 'fff'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1 2024-12-03T23:14:36,233 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,,1733267676090.6d9981bd2d541debb970a37ac121e7ed.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T23:14:36,233 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 6d9981bd2d541debb970a37ac121e7ed, disabling compactions & flushes 2024-12-03T23:14:36,233 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,,1733267676090.6d9981bd2d541debb970a37ac121e7ed. 2024-12-03T23:14:36,233 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,,1733267676090.6d9981bd2d541debb970a37ac121e7ed. 2024-12-03T23:14:36,233 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,,1733267676090.6d9981bd2d541debb970a37ac121e7ed. after waiting 0 ms 2024-12-03T23:14:36,233 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,,1733267676090.6d9981bd2d541debb970a37ac121e7ed. 2024-12-03T23:14:36,233 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,,1733267676090.6d9981bd2d541debb970a37ac121e7ed. 2024-12-03T23:14:36,233 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 6d9981bd2d541debb970a37ac121e7ed: Waiting for close lock at 1733267676233Disabling compacts and flushes for region at 1733267676233Disabling writes for close at 1733267676233Writing region close event to WAL at 1733267676233Closed at 1733267676233 2024-12-03T23:14:36,234 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => f514b27fbd5c6ef451e11852713ea906, NAME => 'test,fff,1733267676090.f514b27fbd5c6ef451e11852713ea906.', STARTKEY => 'fff', ENDKEY => 'ggg'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1 2024-12-03T23:14:36,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45029 is added to blk_1073741843_1019 (size=45) 2024-12-03T23:14:36,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44333 is added to blk_1073741842_1018 (size=45) 2024-12-03T23:14:36,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38883 is added to blk_1073741842_1018 (size=45) 2024-12-03T23:14:36,253 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,fff,1733267676090.f514b27fbd5c6ef451e11852713ea906.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T23:14:36,253 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing f514b27fbd5c6ef451e11852713ea906, disabling compactions & flushes 2024-12-03T23:14:36,253 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,fff,1733267676090.f514b27fbd5c6ef451e11852713ea906. 2024-12-03T23:14:36,253 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,fff,1733267676090.f514b27fbd5c6ef451e11852713ea906. 2024-12-03T23:14:36,253 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,fff,1733267676090.f514b27fbd5c6ef451e11852713ea906. after waiting 0 ms 2024-12-03T23:14:36,253 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,fff,1733267676090.f514b27fbd5c6ef451e11852713ea906. 2024-12-03T23:14:36,253 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,fff,1733267676090.f514b27fbd5c6ef451e11852713ea906. 2024-12-03T23:14:36,253 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for f514b27fbd5c6ef451e11852713ea906: Waiting for close lock at 1733267676253Disabling compacts and flushes for region at 1733267676253Disabling writes for close at 1733267676253Writing region close event to WAL at 1733267676253Closed at 1733267676253 2024-12-03T23:14:36,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44333 is added to blk_1073741843_1019 (size=45) 2024-12-03T23:14:36,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45029 is added to blk_1073741842_1018 (size=45) 2024-12-03T23:14:36,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38883 is added to blk_1073741843_1019 (size=45) 2024-12-03T23:14:36,254 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 73cd6e72f89a0d14d1d979f76c9c07a4, NAME => 'test,ggg,1733267676090.73cd6e72f89a0d14d1d979f76c9c07a4.', STARTKEY => 'ggg', ENDKEY => 'hhh'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1 2024-12-03T23:14:36,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44333 is added to blk_1073741841_1017 (size=45) 2024-12-03T23:14:36,254 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,eee,1733267676090.212a9963c9cddede26a4eb521cff5254.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T23:14:36,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38883 is added to blk_1073741841_1017 (size=45) 2024-12-03T23:14:36,255 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 212a9963c9cddede26a4eb521cff5254, disabling compactions & flushes 2024-12-03T23:14:36,255 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,eee,1733267676090.212a9963c9cddede26a4eb521cff5254. 2024-12-03T23:14:36,255 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,eee,1733267676090.212a9963c9cddede26a4eb521cff5254. 2024-12-03T23:14:36,255 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,eee,1733267676090.212a9963c9cddede26a4eb521cff5254. after waiting 0 ms 2024-12-03T23:14:36,255 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,eee,1733267676090.212a9963c9cddede26a4eb521cff5254. 2024-12-03T23:14:36,255 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,eee,1733267676090.212a9963c9cddede26a4eb521cff5254. 2024-12-03T23:14:36,255 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 212a9963c9cddede26a4eb521cff5254: Waiting for close lock at 1733267676255Disabling compacts and flushes for region at 1733267676255Disabling writes for close at 1733267676255Writing region close event to WAL at 1733267676255Closed at 1733267676255 2024-12-03T23:14:36,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45029 is added to blk_1073741841_1017 (size=45) 2024-12-03T23:14:36,256 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 256869bf1dc8aa33be79adb19e13d134, NAME => 'test,hhh,1733267676090.256869bf1dc8aa33be79adb19e13d134.', STARTKEY => 'hhh', ENDKEY => 'iii'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1 2024-12-03T23:14:36,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45029 is added to blk_1073741844_1020 (size=45) 2024-12-03T23:14:36,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38883 is added to blk_1073741845_1021 (size=45) 2024-12-03T23:14:36,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44333 is added to blk_1073741844_1020 (size=45) 2024-12-03T23:14:36,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45029 is added to blk_1073741845_1021 (size=45) 2024-12-03T23:14:36,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38883 is added to blk_1073741844_1020 (size=45) 2024-12-03T23:14:36,269 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,hhh,1733267676090.256869bf1dc8aa33be79adb19e13d134.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T23:14:36,269 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,ggg,1733267676090.73cd6e72f89a0d14d1d979f76c9c07a4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T23:14:36,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44333 is added to blk_1073741845_1021 (size=45) 2024-12-03T23:14:36,269 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 256869bf1dc8aa33be79adb19e13d134, disabling compactions & flushes 2024-12-03T23:14:36,269 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,hhh,1733267676090.256869bf1dc8aa33be79adb19e13d134. 2024-12-03T23:14:36,269 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 73cd6e72f89a0d14d1d979f76c9c07a4, disabling compactions & flushes 2024-12-03T23:14:36,269 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,hhh,1733267676090.256869bf1dc8aa33be79adb19e13d134. 2024-12-03T23:14:36,269 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,ggg,1733267676090.73cd6e72f89a0d14d1d979f76c9c07a4. 2024-12-03T23:14:36,269 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,hhh,1733267676090.256869bf1dc8aa33be79adb19e13d134. after waiting 0 ms 2024-12-03T23:14:36,269 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,hhh,1733267676090.256869bf1dc8aa33be79adb19e13d134. 2024-12-03T23:14:36,269 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ggg,1733267676090.73cd6e72f89a0d14d1d979f76c9c07a4. 2024-12-03T23:14:36,269 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,hhh,1733267676090.256869bf1dc8aa33be79adb19e13d134. 2024-12-03T23:14:36,269 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,ggg,1733267676090.73cd6e72f89a0d14d1d979f76c9c07a4. after waiting 0 ms 2024-12-03T23:14:36,269 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 256869bf1dc8aa33be79adb19e13d134: Waiting for close lock at 1733267676269Disabling compacts and flushes for region at 1733267676269Disabling writes for close at 1733267676269Writing region close event to WAL at 1733267676269Closed at 1733267676269 2024-12-03T23:14:36,269 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,ggg,1733267676090.73cd6e72f89a0d14d1d979f76c9c07a4. 2024-12-03T23:14:36,270 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,ggg,1733267676090.73cd6e72f89a0d14d1d979f76c9c07a4. 2024-12-03T23:14:36,270 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 73cd6e72f89a0d14d1d979f76c9c07a4: Waiting for close lock at 1733267676269Disabling compacts and flushes for region at 1733267676269Disabling writes for close at 1733267676269Writing region close event to WAL at 1733267676269Closed at 1733267676269 2024-12-03T23:14:36,270 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 2db7cef12ba09a51b0a7204fc81deeec, NAME => 'test,iii,1733267676090.2db7cef12ba09a51b0a7204fc81deeec.', STARTKEY => 'iii', ENDKEY => 'jjj'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1 2024-12-03T23:14:36,271 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 384b2e2791a8bf699cb7996e1aa7469c, NAME => 'test,jjj,1733267676090.384b2e2791a8bf699cb7996e1aa7469c.', STARTKEY => 'jjj', ENDKEY => 'kkk'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1 2024-12-03T23:14:36,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-03T23:14:36,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38883 is added to blk_1073741846_1022 (size=45) 2024-12-03T23:14:36,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45029 is added to blk_1073741846_1022 (size=45) 2024-12-03T23:14:36,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44333 is added to blk_1073741846_1022 (size=45) 2024-12-03T23:14:36,286 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,iii,1733267676090.2db7cef12ba09a51b0a7204fc81deeec.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T23:14:36,286 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 2db7cef12ba09a51b0a7204fc81deeec, disabling compactions & flushes 2024-12-03T23:14:36,286 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,iii,1733267676090.2db7cef12ba09a51b0a7204fc81deeec. 2024-12-03T23:14:36,286 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,iii,1733267676090.2db7cef12ba09a51b0a7204fc81deeec. 2024-12-03T23:14:36,286 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,iii,1733267676090.2db7cef12ba09a51b0a7204fc81deeec. after waiting 0 ms 2024-12-03T23:14:36,286 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,iii,1733267676090.2db7cef12ba09a51b0a7204fc81deeec. 2024-12-03T23:14:36,286 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,iii,1733267676090.2db7cef12ba09a51b0a7204fc81deeec. 2024-12-03T23:14:36,287 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 2db7cef12ba09a51b0a7204fc81deeec: Waiting for close lock at 1733267676286Disabling compacts and flushes for region at 1733267676286Disabling writes for close at 1733267676286Writing region close event to WAL at 1733267676286Closed at 1733267676286 2024-12-03T23:14:36,287 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 5ddac1aa9a6f0af338b83490d6d952c2, NAME => 'test,kkk,1733267676090.5ddac1aa9a6f0af338b83490d6d952c2.', STARTKEY => 'kkk', ENDKEY => 'lll'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1 2024-12-03T23:14:36,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45029 is added to blk_1073741847_1023 (size=45) 2024-12-03T23:14:36,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44333 is added to blk_1073741847_1023 (size=45) 2024-12-03T23:14:36,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38883 is added to blk_1073741847_1023 (size=45) 2024-12-03T23:14:36,290 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,jjj,1733267676090.384b2e2791a8bf699cb7996e1aa7469c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T23:14:36,291 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 384b2e2791a8bf699cb7996e1aa7469c, disabling compactions & flushes 2024-12-03T23:14:36,291 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,jjj,1733267676090.384b2e2791a8bf699cb7996e1aa7469c. 2024-12-03T23:14:36,291 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,jjj,1733267676090.384b2e2791a8bf699cb7996e1aa7469c. 2024-12-03T23:14:36,291 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,jjj,1733267676090.384b2e2791a8bf699cb7996e1aa7469c. after waiting 0 ms 2024-12-03T23:14:36,291 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,jjj,1733267676090.384b2e2791a8bf699cb7996e1aa7469c. 2024-12-03T23:14:36,291 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,jjj,1733267676090.384b2e2791a8bf699cb7996e1aa7469c. 2024-12-03T23:14:36,291 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 384b2e2791a8bf699cb7996e1aa7469c: Waiting for close lock at 1733267676291Disabling compacts and flushes for region at 1733267676291Disabling writes for close at 1733267676291Writing region close event to WAL at 1733267676291Closed at 1733267676291 2024-12-03T23:14:36,291 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 1d026a7a0318e7deb79fa16b1262ae2c, NAME => 'test,lll,1733267676090.1d026a7a0318e7deb79fa16b1262ae2c.', STARTKEY => 'lll', ENDKEY => 'mmm'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1 2024-12-03T23:14:36,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44333 is added to blk_1073741848_1024 (size=45) 2024-12-03T23:14:36,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38883 is added to blk_1073741848_1024 (size=45) 2024-12-03T23:14:36,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45029 is added to blk_1073741848_1024 (size=45) 2024-12-03T23:14:36,300 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,kkk,1733267676090.5ddac1aa9a6f0af338b83490d6d952c2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T23:14:36,300 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 5ddac1aa9a6f0af338b83490d6d952c2, disabling compactions & flushes 2024-12-03T23:14:36,300 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,kkk,1733267676090.5ddac1aa9a6f0af338b83490d6d952c2. 2024-12-03T23:14:36,300 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,kkk,1733267676090.5ddac1aa9a6f0af338b83490d6d952c2. 2024-12-03T23:14:36,300 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,kkk,1733267676090.5ddac1aa9a6f0af338b83490d6d952c2. after waiting 0 ms 2024-12-03T23:14:36,300 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,kkk,1733267676090.5ddac1aa9a6f0af338b83490d6d952c2. 2024-12-03T23:14:36,300 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,kkk,1733267676090.5ddac1aa9a6f0af338b83490d6d952c2. 2024-12-03T23:14:36,301 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 5ddac1aa9a6f0af338b83490d6d952c2: Waiting for close lock at 1733267676300Disabling compacts and flushes for region at 1733267676300Disabling writes for close at 1733267676300Writing region close event to WAL at 1733267676300Closed at 1733267676300 2024-12-03T23:14:36,301 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 4e6706440af7674db97bb48a05858115, NAME => 'test,mmm,1733267676090.4e6706440af7674db97bb48a05858115.', STARTKEY => 'mmm', ENDKEY => 'nnn'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1 2024-12-03T23:14:36,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44333 is added to blk_1073741849_1025 (size=45) 2024-12-03T23:14:36,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45029 is added to blk_1073741849_1025 (size=45) 2024-12-03T23:14:36,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38883 is added to blk_1073741849_1025 (size=45) 2024-12-03T23:14:36,312 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,lll,1733267676090.1d026a7a0318e7deb79fa16b1262ae2c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T23:14:36,312 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 1d026a7a0318e7deb79fa16b1262ae2c, disabling compactions & flushes 2024-12-03T23:14:36,312 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,lll,1733267676090.1d026a7a0318e7deb79fa16b1262ae2c. 2024-12-03T23:14:36,312 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,lll,1733267676090.1d026a7a0318e7deb79fa16b1262ae2c. 2024-12-03T23:14:36,312 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,lll,1733267676090.1d026a7a0318e7deb79fa16b1262ae2c. after waiting 0 ms 2024-12-03T23:14:36,312 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,lll,1733267676090.1d026a7a0318e7deb79fa16b1262ae2c. 2024-12-03T23:14:36,312 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,lll,1733267676090.1d026a7a0318e7deb79fa16b1262ae2c. 2024-12-03T23:14:36,313 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 1d026a7a0318e7deb79fa16b1262ae2c: Waiting for close lock at 1733267676312Disabling compacts and flushes for region at 1733267676312Disabling writes for close at 1733267676312Writing region close event to WAL at 1733267676312Closed at 1733267676312 2024-12-03T23:14:36,313 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => f00ca5d778884039f3248393a629dcfc, NAME => 'test,nnn,1733267676090.f00ca5d778884039f3248393a629dcfc.', STARTKEY => 'nnn', ENDKEY => 'ooo'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1 2024-12-03T23:14:36,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44333 is added to blk_1073741850_1026 (size=45) 2024-12-03T23:14:36,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38883 is added to blk_1073741850_1026 (size=45) 2024-12-03T23:14:36,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45029 is added to blk_1073741850_1026 (size=45) 2024-12-03T23:14:36,324 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,mmm,1733267676090.4e6706440af7674db97bb48a05858115.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T23:14:36,324 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 4e6706440af7674db97bb48a05858115, disabling compactions & flushes 2024-12-03T23:14:36,324 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,mmm,1733267676090.4e6706440af7674db97bb48a05858115. 2024-12-03T23:14:36,325 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,mmm,1733267676090.4e6706440af7674db97bb48a05858115. 2024-12-03T23:14:36,325 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,mmm,1733267676090.4e6706440af7674db97bb48a05858115. after waiting 0 ms 2024-12-03T23:14:36,325 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,mmm,1733267676090.4e6706440af7674db97bb48a05858115. 2024-12-03T23:14:36,325 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,mmm,1733267676090.4e6706440af7674db97bb48a05858115. 2024-12-03T23:14:36,325 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 4e6706440af7674db97bb48a05858115: Waiting for close lock at 1733267676324Disabling compacts and flushes for region at 1733267676324Disabling writes for close at 1733267676325 (+1 ms)Writing region close event to WAL at 1733267676325Closed at 1733267676325 2024-12-03T23:14:36,326 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 5e6925efcdb64a8de08b4c4bbf3a3d2e, NAME => 'test,ooo,1733267676090.5e6925efcdb64a8de08b4c4bbf3a3d2e.', STARTKEY => 'ooo', ENDKEY => 'ppp'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1 2024-12-03T23:14:36,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38883 is added to blk_1073741851_1027 (size=45) 2024-12-03T23:14:36,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44333 is added to blk_1073741851_1027 (size=45) 2024-12-03T23:14:36,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45029 is added to blk_1073741851_1027 (size=45) 2024-12-03T23:14:36,333 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,nnn,1733267676090.f00ca5d778884039f3248393a629dcfc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T23:14:36,333 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing f00ca5d778884039f3248393a629dcfc, disabling compactions & flushes 2024-12-03T23:14:36,333 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,nnn,1733267676090.f00ca5d778884039f3248393a629dcfc. 2024-12-03T23:14:36,333 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,nnn,1733267676090.f00ca5d778884039f3248393a629dcfc. 2024-12-03T23:14:36,333 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,nnn,1733267676090.f00ca5d778884039f3248393a629dcfc. after waiting 0 ms 2024-12-03T23:14:36,334 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,nnn,1733267676090.f00ca5d778884039f3248393a629dcfc. 2024-12-03T23:14:36,334 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,nnn,1733267676090.f00ca5d778884039f3248393a629dcfc. 2024-12-03T23:14:36,334 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for f00ca5d778884039f3248393a629dcfc: Waiting for close lock at 1733267676333Disabling compacts and flushes for region at 1733267676333Disabling writes for close at 1733267676334 (+1 ms)Writing region close event to WAL at 1733267676334Closed at 1733267676334 2024-12-03T23:14:36,335 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 742c43a49ea3b763b8232c8e8c8966b3, NAME => 'test,ppp,1733267676090.742c43a49ea3b763b8232c8e8c8966b3.', STARTKEY => 'ppp', ENDKEY => 'qqq'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1 2024-12-03T23:14:36,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44333 is added to blk_1073741852_1028 (size=45) 2024-12-03T23:14:36,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38883 is added to blk_1073741852_1028 (size=45) 2024-12-03T23:14:36,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45029 is added to blk_1073741852_1028 (size=45) 2024-12-03T23:14:36,339 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,ooo,1733267676090.5e6925efcdb64a8de08b4c4bbf3a3d2e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T23:14:36,339 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 5e6925efcdb64a8de08b4c4bbf3a3d2e, disabling compactions & flushes 2024-12-03T23:14:36,339 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,ooo,1733267676090.5e6925efcdb64a8de08b4c4bbf3a3d2e. 2024-12-03T23:14:36,340 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ooo,1733267676090.5e6925efcdb64a8de08b4c4bbf3a3d2e. 2024-12-03T23:14:36,340 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,ooo,1733267676090.5e6925efcdb64a8de08b4c4bbf3a3d2e. after waiting 0 ms 2024-12-03T23:14:36,340 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,ooo,1733267676090.5e6925efcdb64a8de08b4c4bbf3a3d2e. 2024-12-03T23:14:36,340 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,ooo,1733267676090.5e6925efcdb64a8de08b4c4bbf3a3d2e. 2024-12-03T23:14:36,340 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 5e6925efcdb64a8de08b4c4bbf3a3d2e: Waiting for close lock at 1733267676339Disabling compacts and flushes for region at 1733267676339Disabling writes for close at 1733267676340 (+1 ms)Writing region close event to WAL at 1733267676340Closed at 1733267676340 2024-12-03T23:14:36,341 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 10beaf4047510d958e4a559194242a57, NAME => 'test,qqq,1733267676090.10beaf4047510d958e4a559194242a57.', STARTKEY => 'qqq', ENDKEY => 'rrr'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1 2024-12-03T23:14:36,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38883 is added to blk_1073741853_1029 (size=45) 2024-12-03T23:14:36,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45029 is added to blk_1073741853_1029 (size=45) 2024-12-03T23:14:36,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44333 is added to blk_1073741853_1029 (size=45) 2024-12-03T23:14:36,349 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,ppp,1733267676090.742c43a49ea3b763b8232c8e8c8966b3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T23:14:36,349 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 742c43a49ea3b763b8232c8e8c8966b3, disabling compactions & flushes 2024-12-03T23:14:36,349 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,ppp,1733267676090.742c43a49ea3b763b8232c8e8c8966b3. 2024-12-03T23:14:36,349 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ppp,1733267676090.742c43a49ea3b763b8232c8e8c8966b3. 2024-12-03T23:14:36,349 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,ppp,1733267676090.742c43a49ea3b763b8232c8e8c8966b3. after waiting 0 ms 2024-12-03T23:14:36,349 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,ppp,1733267676090.742c43a49ea3b763b8232c8e8c8966b3. 2024-12-03T23:14:36,349 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,ppp,1733267676090.742c43a49ea3b763b8232c8e8c8966b3. 2024-12-03T23:14:36,349 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 742c43a49ea3b763b8232c8e8c8966b3: Waiting for close lock at 1733267676349Disabling compacts and flushes for region at 1733267676349Disabling writes for close at 1733267676349Writing region close event to WAL at 1733267676349Closed at 1733267676349 2024-12-03T23:14:36,350 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 89a698454f10c9fb6cfc108c35d8415a, NAME => 'test,rrr,1733267676090.89a698454f10c9fb6cfc108c35d8415a.', STARTKEY => 'rrr', ENDKEY => 'sss'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1 2024-12-03T23:14:36,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45029 is added to blk_1073741854_1030 (size=45) 2024-12-03T23:14:36,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44333 is added to blk_1073741854_1030 (size=45) 2024-12-03T23:14:36,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38883 is added to blk_1073741854_1030 (size=45) 2024-12-03T23:14:36,355 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,qqq,1733267676090.10beaf4047510d958e4a559194242a57.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T23:14:36,355 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 10beaf4047510d958e4a559194242a57, disabling compactions & flushes 2024-12-03T23:14:36,355 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,qqq,1733267676090.10beaf4047510d958e4a559194242a57. 2024-12-03T23:14:36,355 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,qqq,1733267676090.10beaf4047510d958e4a559194242a57. 2024-12-03T23:14:36,355 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,qqq,1733267676090.10beaf4047510d958e4a559194242a57. after waiting 0 ms 2024-12-03T23:14:36,355 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,qqq,1733267676090.10beaf4047510d958e4a559194242a57. 2024-12-03T23:14:36,355 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,qqq,1733267676090.10beaf4047510d958e4a559194242a57. 2024-12-03T23:14:36,355 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 10beaf4047510d958e4a559194242a57: Waiting for close lock at 1733267676355Disabling compacts and flushes for region at 1733267676355Disabling writes for close at 1733267676355Writing region close event to WAL at 1733267676355Closed at 1733267676355 2024-12-03T23:14:36,356 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 2b7da3736c6b257b43fe66ea13807bc6, NAME => 'test,sss,1733267676090.2b7da3736c6b257b43fe66ea13807bc6.', STARTKEY => 'sss', ENDKEY => 'ttt'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1 2024-12-03T23:14:36,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45029 is added to blk_1073741855_1031 (size=45) 2024-12-03T23:14:36,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45029 is added to blk_1073741856_1032 (size=45) 2024-12-03T23:14:36,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38883 is added to blk_1073741856_1032 (size=45) 2024-12-03T23:14:36,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44333 is added to blk_1073741855_1031 (size=45) 2024-12-03T23:14:36,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38883 is added to blk_1073741855_1031 (size=45) 2024-12-03T23:14:36,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44333 is added to blk_1073741856_1032 (size=45) 2024-12-03T23:14:36,371 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,rrr,1733267676090.89a698454f10c9fb6cfc108c35d8415a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T23:14:36,371 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 89a698454f10c9fb6cfc108c35d8415a, disabling compactions & flushes 2024-12-03T23:14:36,372 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,rrr,1733267676090.89a698454f10c9fb6cfc108c35d8415a. 2024-12-03T23:14:36,372 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,rrr,1733267676090.89a698454f10c9fb6cfc108c35d8415a. 2024-12-03T23:14:36,372 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,rrr,1733267676090.89a698454f10c9fb6cfc108c35d8415a. after waiting 0 ms 2024-12-03T23:14:36,372 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,sss,1733267676090.2b7da3736c6b257b43fe66ea13807bc6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T23:14:36,372 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,rrr,1733267676090.89a698454f10c9fb6cfc108c35d8415a. 2024-12-03T23:14:36,372 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,rrr,1733267676090.89a698454f10c9fb6cfc108c35d8415a. 2024-12-03T23:14:36,372 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 89a698454f10c9fb6cfc108c35d8415a: Waiting for close lock at 1733267676371Disabling compacts and flushes for region at 1733267676371Disabling writes for close at 1733267676372 (+1 ms)Writing region close event to WAL at 1733267676372Closed at 1733267676372 2024-12-03T23:14:36,372 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 2b7da3736c6b257b43fe66ea13807bc6, disabling compactions & flushes 2024-12-03T23:14:36,372 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,sss,1733267676090.2b7da3736c6b257b43fe66ea13807bc6. 2024-12-03T23:14:36,372 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a6de975926600fad463e13238b2ac8a9, NAME => 'test,ttt,1733267676090.a6de975926600fad463e13238b2ac8a9.', STARTKEY => 'ttt', ENDKEY => 'uuu'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1 2024-12-03T23:14:36,372 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,sss,1733267676090.2b7da3736c6b257b43fe66ea13807bc6. 2024-12-03T23:14:36,373 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,sss,1733267676090.2b7da3736c6b257b43fe66ea13807bc6. after waiting 0 ms 2024-12-03T23:14:36,373 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,sss,1733267676090.2b7da3736c6b257b43fe66ea13807bc6. 2024-12-03T23:14:36,373 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,sss,1733267676090.2b7da3736c6b257b43fe66ea13807bc6. 2024-12-03T23:14:36,373 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 2b7da3736c6b257b43fe66ea13807bc6: Waiting for close lock at 1733267676372Disabling compacts and flushes for region at 1733267676372Disabling writes for close at 1733267676373 (+1 ms)Writing region close event to WAL at 1733267676373Closed at 1733267676373 2024-12-03T23:14:36,374 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => e5e4ecc042f874ec5567f7e86548256f, NAME => 'test,uuu,1733267676090.e5e4ecc042f874ec5567f7e86548256f.', STARTKEY => 'uuu', ENDKEY => 'vvv'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1 2024-12-03T23:14:36,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38883 is added to blk_1073741858_1034 (size=45) 2024-12-03T23:14:36,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44333 is added to blk_1073741858_1034 (size=45) 2024-12-03T23:14:36,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45029 is added to blk_1073741858_1034 (size=45) 2024-12-03T23:14:36,396 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,ttt,1733267676090.a6de975926600fad463e13238b2ac8a9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T23:14:36,397 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing a6de975926600fad463e13238b2ac8a9, disabling compactions & flushes 2024-12-03T23:14:36,397 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,ttt,1733267676090.a6de975926600fad463e13238b2ac8a9. 2024-12-03T23:14:36,397 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ttt,1733267676090.a6de975926600fad463e13238b2ac8a9. 2024-12-03T23:14:36,397 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,ttt,1733267676090.a6de975926600fad463e13238b2ac8a9. after waiting 0 ms 2024-12-03T23:14:36,397 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,ttt,1733267676090.a6de975926600fad463e13238b2ac8a9. 2024-12-03T23:14:36,397 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,ttt,1733267676090.a6de975926600fad463e13238b2ac8a9. 2024-12-03T23:14:36,397 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for a6de975926600fad463e13238b2ac8a9: Waiting for close lock at 1733267676396Disabling compacts and flushes for region at 1733267676396Disabling writes for close at 1733267676397 (+1 ms)Writing region close event to WAL at 1733267676397Closed at 1733267676397 2024-12-03T23:14:36,398 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 23a123891f32f583441cd7569a6de1f2, NAME => 'test,vvv,1733267676090.23a123891f32f583441cd7569a6de1f2.', STARTKEY => 'vvv', ENDKEY => 'www'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1 2024-12-03T23:14:36,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45029 is added to blk_1073741857_1033 (size=45) 2024-12-03T23:14:36,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38883 is added to blk_1073741857_1033 (size=45) 2024-12-03T23:14:36,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44333 is added to blk_1073741857_1033 (size=45) 2024-12-03T23:14:36,405 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,uuu,1733267676090.e5e4ecc042f874ec5567f7e86548256f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T23:14:36,405 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing e5e4ecc042f874ec5567f7e86548256f, disabling compactions & flushes 2024-12-03T23:14:36,405 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,uuu,1733267676090.e5e4ecc042f874ec5567f7e86548256f. 2024-12-03T23:14:36,405 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,uuu,1733267676090.e5e4ecc042f874ec5567f7e86548256f. 2024-12-03T23:14:36,405 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,uuu,1733267676090.e5e4ecc042f874ec5567f7e86548256f. after waiting 0 ms 2024-12-03T23:14:36,405 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,uuu,1733267676090.e5e4ecc042f874ec5567f7e86548256f. 2024-12-03T23:14:36,405 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,uuu,1733267676090.e5e4ecc042f874ec5567f7e86548256f. 2024-12-03T23:14:36,405 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for e5e4ecc042f874ec5567f7e86548256f: Waiting for close lock at 1733267676405Disabling compacts and flushes for region at 1733267676405Disabling writes for close at 1733267676405Writing region close event to WAL at 1733267676405Closed at 1733267676405 2024-12-03T23:14:36,406 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => a0f09ea7cbe3ceec5232d11c21a512a4, NAME => 'test,www,1733267676090.a0f09ea7cbe3ceec5232d11c21a512a4.', STARTKEY => 'www', ENDKEY => 'xxx'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1 2024-12-03T23:14:36,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45029 is added to blk_1073741859_1035 (size=45) 2024-12-03T23:14:36,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44333 is added to blk_1073741859_1035 (size=45) 2024-12-03T23:14:36,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38883 is added to blk_1073741859_1035 (size=45) 2024-12-03T23:14:36,414 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,vvv,1733267676090.23a123891f32f583441cd7569a6de1f2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T23:14:36,414 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 23a123891f32f583441cd7569a6de1f2, disabling compactions & flushes 2024-12-03T23:14:36,414 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,vvv,1733267676090.23a123891f32f583441cd7569a6de1f2. 2024-12-03T23:14:36,414 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,vvv,1733267676090.23a123891f32f583441cd7569a6de1f2. 2024-12-03T23:14:36,414 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,vvv,1733267676090.23a123891f32f583441cd7569a6de1f2. after waiting 0 ms 2024-12-03T23:14:36,414 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,vvv,1733267676090.23a123891f32f583441cd7569a6de1f2. 2024-12-03T23:14:36,414 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,vvv,1733267676090.23a123891f32f583441cd7569a6de1f2. 2024-12-03T23:14:36,414 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 23a123891f32f583441cd7569a6de1f2: Waiting for close lock at 1733267676414Disabling compacts and flushes for region at 1733267676414Disabling writes for close at 1733267676414Writing region close event to WAL at 1733267676414Closed at 1733267676414 2024-12-03T23:14:36,415 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e96b12de28d6c3047803f54a3b027459, NAME => 'test,xxx,1733267676090.e96b12de28d6c3047803f54a3b027459.', STARTKEY => 'xxx', ENDKEY => 'yyy'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1 2024-12-03T23:14:36,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38883 is added to blk_1073741860_1036 (size=45) 2024-12-03T23:14:36,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44333 is added to blk_1073741860_1036 (size=45) 2024-12-03T23:14:36,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45029 is added to blk_1073741860_1036 (size=45) 2024-12-03T23:14:36,418 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,www,1733267676090.a0f09ea7cbe3ceec5232d11c21a512a4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T23:14:36,418 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing a0f09ea7cbe3ceec5232d11c21a512a4, disabling compactions & flushes 2024-12-03T23:14:36,418 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,www,1733267676090.a0f09ea7cbe3ceec5232d11c21a512a4. 2024-12-03T23:14:36,418 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,www,1733267676090.a0f09ea7cbe3ceec5232d11c21a512a4. 2024-12-03T23:14:36,418 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,www,1733267676090.a0f09ea7cbe3ceec5232d11c21a512a4. after waiting 0 ms 2024-12-03T23:14:36,418 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,www,1733267676090.a0f09ea7cbe3ceec5232d11c21a512a4. 2024-12-03T23:14:36,418 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,www,1733267676090.a0f09ea7cbe3ceec5232d11c21a512a4. 2024-12-03T23:14:36,418 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for a0f09ea7cbe3ceec5232d11c21a512a4: Waiting for close lock at 1733267676418Disabling compacts and flushes for region at 1733267676418Disabling writes for close at 1733267676418Writing region close event to WAL at 1733267676418Closed at 1733267676418 2024-12-03T23:14:36,419 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 553e1b6e96399dd43d3b8f7cfad54903, NAME => 'test,yyy,1733267676090.553e1b6e96399dd43d3b8f7cfad54903.', STARTKEY => 'yyy', ENDKEY => 'zzz'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1 2024-12-03T23:14:36,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38883 is added to blk_1073741861_1037 (size=45) 2024-12-03T23:14:36,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44333 is added to blk_1073741861_1037 (size=45) 2024-12-03T23:14:36,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45029 is added to blk_1073741861_1037 (size=45) 2024-12-03T23:14:36,428 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,xxx,1733267676090.e96b12de28d6c3047803f54a3b027459.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T23:14:36,429 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing e96b12de28d6c3047803f54a3b027459, disabling compactions & flushes 2024-12-03T23:14:36,429 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,xxx,1733267676090.e96b12de28d6c3047803f54a3b027459. 2024-12-03T23:14:36,429 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,xxx,1733267676090.e96b12de28d6c3047803f54a3b027459. 2024-12-03T23:14:36,429 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,xxx,1733267676090.e96b12de28d6c3047803f54a3b027459. after waiting 0 ms 2024-12-03T23:14:36,429 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,xxx,1733267676090.e96b12de28d6c3047803f54a3b027459. 2024-12-03T23:14:36,429 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,xxx,1733267676090.e96b12de28d6c3047803f54a3b027459. 2024-12-03T23:14:36,429 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for e96b12de28d6c3047803f54a3b027459: Waiting for close lock at 1733267676428Disabling compacts and flushes for region at 1733267676428Disabling writes for close at 1733267676429 (+1 ms)Writing region close event to WAL at 1733267676429Closed at 1733267676429 2024-12-03T23:14:36,429 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => db9924063978f5acb21b073c5d94a7ad, NAME => 'test,zzz,1733267676090.db9924063978f5acb21b073c5d94a7ad.', STARTKEY => 'zzz', ENDKEY => ''}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1 2024-12-03T23:14:36,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44333 is added to blk_1073741862_1038 (size=45) 2024-12-03T23:14:36,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45029 is added to blk_1073741862_1038 (size=45) 2024-12-03T23:14:36,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38883 is added to blk_1073741862_1038 (size=45) 2024-12-03T23:14:36,433 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,yyy,1733267676090.553e1b6e96399dd43d3b8f7cfad54903.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T23:14:36,433 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 553e1b6e96399dd43d3b8f7cfad54903, disabling compactions & flushes 2024-12-03T23:14:36,433 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,yyy,1733267676090.553e1b6e96399dd43d3b8f7cfad54903. 2024-12-03T23:14:36,434 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,yyy,1733267676090.553e1b6e96399dd43d3b8f7cfad54903. 2024-12-03T23:14:36,434 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,yyy,1733267676090.553e1b6e96399dd43d3b8f7cfad54903. after waiting 0 ms 2024-12-03T23:14:36,434 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,yyy,1733267676090.553e1b6e96399dd43d3b8f7cfad54903. 2024-12-03T23:14:36,434 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,yyy,1733267676090.553e1b6e96399dd43d3b8f7cfad54903. 2024-12-03T23:14:36,434 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 553e1b6e96399dd43d3b8f7cfad54903: Waiting for close lock at 1733267676433Disabling compacts and flushes for region at 1733267676433Disabling writes for close at 1733267676434 (+1 ms)Writing region close event to WAL at 1733267676434Closed at 1733267676434 2024-12-03T23:14:36,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38883 is added to blk_1073741863_1039 (size=42) 2024-12-03T23:14:36,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45029 is added to blk_1073741863_1039 (size=42) 2024-12-03T23:14:36,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44333 is added to blk_1073741863_1039 (size=42) 2024-12-03T23:14:36,444 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,zzz,1733267676090.db9924063978f5acb21b073c5d94a7ad.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T23:14:36,444 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing db9924063978f5acb21b073c5d94a7ad, disabling compactions & flushes 2024-12-03T23:14:36,444 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,zzz,1733267676090.db9924063978f5acb21b073c5d94a7ad. 2024-12-03T23:14:36,444 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,zzz,1733267676090.db9924063978f5acb21b073c5d94a7ad. 2024-12-03T23:14:36,444 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,zzz,1733267676090.db9924063978f5acb21b073c5d94a7ad. after waiting 0 ms 2024-12-03T23:14:36,444 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,zzz,1733267676090.db9924063978f5acb21b073c5d94a7ad. 2024-12-03T23:14:36,444 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,zzz,1733267676090.db9924063978f5acb21b073c5d94a7ad. 2024-12-03T23:14:36,444 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for db9924063978f5acb21b073c5d94a7ad: Waiting for close lock at 1733267676444Disabling compacts and flushes for region at 1733267676444Disabling writes for close at 1733267676444Writing region close event to WAL at 1733267676444Closed at 1733267676444 2024-12-03T23:14:36,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-03T23:14:36,655 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ddd,1733267676090.55eff67791415c655cef877182b9bbe1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T23:14:36,656 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 55eff67791415c655cef877182b9bbe1, disabling compactions & flushes 2024-12-03T23:14:36,656 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ddd,1733267676090.55eff67791415c655cef877182b9bbe1. 2024-12-03T23:14:36,656 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ddd,1733267676090.55eff67791415c655cef877182b9bbe1. 2024-12-03T23:14:36,656 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ddd,1733267676090.55eff67791415c655cef877182b9bbe1. after waiting 0 ms 2024-12-03T23:14:36,656 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ddd,1733267676090.55eff67791415c655cef877182b9bbe1. 2024-12-03T23:14:36,656 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ddd,1733267676090.55eff67791415c655cef877182b9bbe1. 2024-12-03T23:14:36,656 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 55eff67791415c655cef877182b9bbe1: Waiting for close lock at 1733267676656Disabling compacts and flushes for region at 1733267676656Disabling writes for close at 1733267676656Writing region close event to WAL at 1733267676656Closed at 1733267676656 2024-12-03T23:14:36,659 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_ADD_TO_META 2024-12-03T23:14:36,664 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ccc,1733267676090.0e35e57b6f6cbff01d3e782fbf6efd1f.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733267676659"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733267676659"}]},"ts":"1733267676659"} 2024-12-03T23:14:36,664 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,bbb,1733267676090.f71f90ef2d2f59e885f2ad6ca1a99596.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733267676659"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733267676659"}]},"ts":"1733267676659"} 2024-12-03T23:14:36,664 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,,1733267676090.6d9981bd2d541debb970a37ac121e7ed.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1733267676659"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733267676659"}]},"ts":"1733267676659"} 2024-12-03T23:14:36,665 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,fff,1733267676090.f514b27fbd5c6ef451e11852713ea906.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733267676659"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733267676659"}]},"ts":"1733267676659"} 2024-12-03T23:14:36,665 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,eee,1733267676090.212a9963c9cddede26a4eb521cff5254.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733267676659"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733267676659"}]},"ts":"1733267676659"} 2024-12-03T23:14:36,665 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,hhh,1733267676090.256869bf1dc8aa33be79adb19e13d134.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733267676659"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733267676659"}]},"ts":"1733267676659"} 2024-12-03T23:14:36,665 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ggg,1733267676090.73cd6e72f89a0d14d1d979f76c9c07a4.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733267676659"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733267676659"}]},"ts":"1733267676659"} 2024-12-03T23:14:36,665 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,iii,1733267676090.2db7cef12ba09a51b0a7204fc81deeec.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733267676659"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733267676659"}]},"ts":"1733267676659"} 2024-12-03T23:14:36,665 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,jjj,1733267676090.384b2e2791a8bf699cb7996e1aa7469c.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733267676659"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733267676659"}]},"ts":"1733267676659"} 2024-12-03T23:14:36,665 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,kkk,1733267676090.5ddac1aa9a6f0af338b83490d6d952c2.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733267676659"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733267676659"}]},"ts":"1733267676659"} 2024-12-03T23:14:36,666 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,lll,1733267676090.1d026a7a0318e7deb79fa16b1262ae2c.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733267676659"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733267676659"}]},"ts":"1733267676659"} 2024-12-03T23:14:36,666 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,mmm,1733267676090.4e6706440af7674db97bb48a05858115.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733267676659"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733267676659"}]},"ts":"1733267676659"} 2024-12-03T23:14:36,666 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,nnn,1733267676090.f00ca5d778884039f3248393a629dcfc.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733267676659"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733267676659"}]},"ts":"1733267676659"} 2024-12-03T23:14:36,666 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ooo,1733267676090.5e6925efcdb64a8de08b4c4bbf3a3d2e.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733267676659"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733267676659"}]},"ts":"1733267676659"} 2024-12-03T23:14:36,666 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ppp,1733267676090.742c43a49ea3b763b8232c8e8c8966b3.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733267676659"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733267676659"}]},"ts":"1733267676659"} 2024-12-03T23:14:36,666 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,qqq,1733267676090.10beaf4047510d958e4a559194242a57.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733267676659"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733267676659"}]},"ts":"1733267676659"} 2024-12-03T23:14:36,666 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,rrr,1733267676090.89a698454f10c9fb6cfc108c35d8415a.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733267676659"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733267676659"}]},"ts":"1733267676659"} 2024-12-03T23:14:36,666 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,sss,1733267676090.2b7da3736c6b257b43fe66ea13807bc6.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733267676659"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733267676659"}]},"ts":"1733267676659"} 2024-12-03T23:14:36,666 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ttt,1733267676090.a6de975926600fad463e13238b2ac8a9.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733267676659"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733267676659"}]},"ts":"1733267676659"} 2024-12-03T23:14:36,666 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,uuu,1733267676090.e5e4ecc042f874ec5567f7e86548256f.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733267676659"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733267676659"}]},"ts":"1733267676659"} 2024-12-03T23:14:36,667 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,vvv,1733267676090.23a123891f32f583441cd7569a6de1f2.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733267676659"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733267676659"}]},"ts":"1733267676659"} 2024-12-03T23:14:36,667 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,www,1733267676090.a0f09ea7cbe3ceec5232d11c21a512a4.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733267676659"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733267676659"}]},"ts":"1733267676659"} 2024-12-03T23:14:36,667 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,xxx,1733267676090.e96b12de28d6c3047803f54a3b027459.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733267676659"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733267676659"}]},"ts":"1733267676659"} 2024-12-03T23:14:36,667 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,yyy,1733267676090.553e1b6e96399dd43d3b8f7cfad54903.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733267676659"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733267676659"}]},"ts":"1733267676659"} 2024-12-03T23:14:36,667 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,zzz,1733267676090.db9924063978f5acb21b073c5d94a7ad.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1733267676659"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733267676659"}]},"ts":"1733267676659"} 2024-12-03T23:14:36,667 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ddd,1733267676090.55eff67791415c655cef877182b9bbe1.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733267676659"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733267676659"}]},"ts":"1733267676659"} 2024-12-03T23:14:36,712 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 26 regions to meta. 2024-12-03T23:14:36,714 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-03T23:14:36,717 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733267676715"}]},"ts":"1733267676715"} 2024-12-03T23:14:36,722 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=test, state=ENABLING in hbase:meta 2024-12-03T23:14:36,722 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(204): Hosts are {5f9d64419733=0} racks are {/default-rack=0} 2024-12-03T23:14:36,740 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-03T23:14:36,740 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-03T23:14:36,740 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-12-03T23:14:36,740 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-03T23:14:36,740 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-03T23:14:36,740 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-12-03T23:14:36,740 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-03T23:14:36,740 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-03T23:14:36,740 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-12-03T23:14:36,740 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-03T23:14:36,742 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=6d9981bd2d541debb970a37ac121e7ed, ASSIGN}, {pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f71f90ef2d2f59e885f2ad6ca1a99596, ASSIGN}, {pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=0e35e57b6f6cbff01d3e782fbf6efd1f, ASSIGN}, {pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=55eff67791415c655cef877182b9bbe1, ASSIGN}, {pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=212a9963c9cddede26a4eb521cff5254, ASSIGN}, {pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f514b27fbd5c6ef451e11852713ea906, ASSIGN}, {pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=73cd6e72f89a0d14d1d979f76c9c07a4, ASSIGN}, {pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=256869bf1dc8aa33be79adb19e13d134, ASSIGN}, {pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2db7cef12ba09a51b0a7204fc81deeec, ASSIGN}, {pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=384b2e2791a8bf699cb7996e1aa7469c, ASSIGN}, {pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=5ddac1aa9a6f0af338b83490d6d952c2, ASSIGN}, {pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=1d026a7a0318e7deb79fa16b1262ae2c, ASSIGN}, {pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4e6706440af7674db97bb48a05858115, ASSIGN}, {pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f00ca5d778884039f3248393a629dcfc, ASSIGN}, {pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=5e6925efcdb64a8de08b4c4bbf3a3d2e, ASSIGN}, {pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=742c43a49ea3b763b8232c8e8c8966b3, ASSIGN}, {pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=10beaf4047510d958e4a559194242a57, ASSIGN}, {pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=89a698454f10c9fb6cfc108c35d8415a, ASSIGN}, {pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2b7da3736c6b257b43fe66ea13807bc6, ASSIGN}, {pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a6de975926600fad463e13238b2ac8a9, ASSIGN}, {pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=e5e4ecc042f874ec5567f7e86548256f, ASSIGN}, {pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=23a123891f32f583441cd7569a6de1f2, ASSIGN}, {pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a0f09ea7cbe3ceec5232d11c21a512a4, ASSIGN}, {pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=e96b12de28d6c3047803f54a3b027459, ASSIGN}, {pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=553e1b6e96399dd43d3b8f7cfad54903, ASSIGN}, {pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=db9924063978f5acb21b073c5d94a7ad, ASSIGN}] 2024-12-03T23:14:36,750 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=db9924063978f5acb21b073c5d94a7ad, ASSIGN 2024-12-03T23:14:36,752 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=db9924063978f5acb21b073c5d94a7ad, ASSIGN; state=OFFLINE, location=5f9d64419733,44899,1733267673202; forceNewPlan=false, retain=false 2024-12-03T23:14:36,753 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=553e1b6e96399dd43d3b8f7cfad54903, ASSIGN 2024-12-03T23:14:36,754 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=553e1b6e96399dd43d3b8f7cfad54903, ASSIGN; state=OFFLINE, location=5f9d64419733,46833,1733267673070; forceNewPlan=false, retain=false 2024-12-03T23:14:36,756 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=e96b12de28d6c3047803f54a3b027459, ASSIGN 2024-12-03T23:14:36,757 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=e96b12de28d6c3047803f54a3b027459, ASSIGN; state=OFFLINE, location=5f9d64419733,41167,1733267673270; forceNewPlan=false, retain=false 2024-12-03T23:14:36,759 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a0f09ea7cbe3ceec5232d11c21a512a4, ASSIGN 2024-12-03T23:14:36,760 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=a0f09ea7cbe3ceec5232d11c21a512a4, ASSIGN; state=OFFLINE, location=5f9d64419733,44899,1733267673202; forceNewPlan=false, retain=false 2024-12-03T23:14:36,761 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=23a123891f32f583441cd7569a6de1f2, ASSIGN 2024-12-03T23:14:36,763 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=23a123891f32f583441cd7569a6de1f2, ASSIGN; state=OFFLINE, location=5f9d64419733,46833,1733267673070; forceNewPlan=false, retain=false 2024-12-03T23:14:36,764 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=e5e4ecc042f874ec5567f7e86548256f, ASSIGN 2024-12-03T23:14:36,765 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=e5e4ecc042f874ec5567f7e86548256f, ASSIGN; state=OFFLINE, location=5f9d64419733,41167,1733267673270; forceNewPlan=false, retain=false 2024-12-03T23:14:36,767 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a6de975926600fad463e13238b2ac8a9, ASSIGN 2024-12-03T23:14:36,768 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=a6de975926600fad463e13238b2ac8a9, ASSIGN; state=OFFLINE, location=5f9d64419733,44899,1733267673202; forceNewPlan=false, retain=false 2024-12-03T23:14:36,769 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2b7da3736c6b257b43fe66ea13807bc6, ASSIGN 2024-12-03T23:14:36,771 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=2b7da3736c6b257b43fe66ea13807bc6, ASSIGN; state=OFFLINE, location=5f9d64419733,46833,1733267673070; forceNewPlan=false, retain=false 2024-12-03T23:14:36,772 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=89a698454f10c9fb6cfc108c35d8415a, ASSIGN 2024-12-03T23:14:36,773 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=89a698454f10c9fb6cfc108c35d8415a, ASSIGN; state=OFFLINE, location=5f9d64419733,41167,1733267673270; forceNewPlan=false, retain=false 2024-12-03T23:14:36,774 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=10beaf4047510d958e4a559194242a57, ASSIGN 2024-12-03T23:14:36,776 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=10beaf4047510d958e4a559194242a57, ASSIGN; state=OFFLINE, location=5f9d64419733,44899,1733267673202; forceNewPlan=false, retain=false 2024-12-03T23:14:36,777 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=742c43a49ea3b763b8232c8e8c8966b3, ASSIGN 2024-12-03T23:14:36,778 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=742c43a49ea3b763b8232c8e8c8966b3, ASSIGN; state=OFFLINE, location=5f9d64419733,46833,1733267673070; forceNewPlan=false, retain=false 2024-12-03T23:14:36,779 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=5e6925efcdb64a8de08b4c4bbf3a3d2e, ASSIGN 2024-12-03T23:14:36,780 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=5e6925efcdb64a8de08b4c4bbf3a3d2e, ASSIGN; state=OFFLINE, location=5f9d64419733,41167,1733267673270; forceNewPlan=false, retain=false 2024-12-03T23:14:36,781 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f00ca5d778884039f3248393a629dcfc, ASSIGN 2024-12-03T23:14:36,783 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=f00ca5d778884039f3248393a629dcfc, ASSIGN; state=OFFLINE, location=5f9d64419733,44899,1733267673202; forceNewPlan=false, retain=false 2024-12-03T23:14:36,784 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4e6706440af7674db97bb48a05858115, ASSIGN 2024-12-03T23:14:36,785 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=4e6706440af7674db97bb48a05858115, ASSIGN; state=OFFLINE, location=5f9d64419733,46833,1733267673070; forceNewPlan=false, retain=false 2024-12-03T23:14:36,787 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=1d026a7a0318e7deb79fa16b1262ae2c, ASSIGN 2024-12-03T23:14:36,788 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=1d026a7a0318e7deb79fa16b1262ae2c, ASSIGN; state=OFFLINE, location=5f9d64419733,41167,1733267673270; forceNewPlan=false, retain=false 2024-12-03T23:14:36,789 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=5ddac1aa9a6f0af338b83490d6d952c2, ASSIGN 2024-12-03T23:14:36,792 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=5ddac1aa9a6f0af338b83490d6d952c2, ASSIGN; state=OFFLINE, location=5f9d64419733,44899,1733267673202; forceNewPlan=false, retain=false 2024-12-03T23:14:36,794 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=384b2e2791a8bf699cb7996e1aa7469c, ASSIGN 2024-12-03T23:14:36,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-03T23:14:36,795 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=384b2e2791a8bf699cb7996e1aa7469c, ASSIGN; state=OFFLINE, location=5f9d64419733,46833,1733267673070; forceNewPlan=false, retain=false 2024-12-03T23:14:36,797 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2db7cef12ba09a51b0a7204fc81deeec, ASSIGN 2024-12-03T23:14:36,798 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=2db7cef12ba09a51b0a7204fc81deeec, ASSIGN; state=OFFLINE, location=5f9d64419733,41167,1733267673270; forceNewPlan=false, retain=false 2024-12-03T23:14:36,800 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=256869bf1dc8aa33be79adb19e13d134, ASSIGN 2024-12-03T23:14:36,801 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=256869bf1dc8aa33be79adb19e13d134, ASSIGN; state=OFFLINE, location=5f9d64419733,46833,1733267673070; forceNewPlan=false, retain=false 2024-12-03T23:14:36,802 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=73cd6e72f89a0d14d1d979f76c9c07a4, ASSIGN 2024-12-03T23:14:36,804 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=73cd6e72f89a0d14d1d979f76c9c07a4, ASSIGN; state=OFFLINE, location=5f9d64419733,44899,1733267673202; forceNewPlan=false, retain=false 2024-12-03T23:14:36,805 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f514b27fbd5c6ef451e11852713ea906, ASSIGN 2024-12-03T23:14:36,807 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=f514b27fbd5c6ef451e11852713ea906, ASSIGN; state=OFFLINE, location=5f9d64419733,44899,1733267673202; forceNewPlan=false, retain=false 2024-12-03T23:14:36,810 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=212a9963c9cddede26a4eb521cff5254, ASSIGN 2024-12-03T23:14:36,812 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=212a9963c9cddede26a4eb521cff5254, ASSIGN; state=OFFLINE, location=5f9d64419733,41167,1733267673270; forceNewPlan=false, retain=false 2024-12-03T23:14:36,813 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=55eff67791415c655cef877182b9bbe1, ASSIGN 2024-12-03T23:14:36,816 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=55eff67791415c655cef877182b9bbe1, ASSIGN; state=OFFLINE, location=5f9d64419733,41167,1733267673270; forceNewPlan=false, retain=false 2024-12-03T23:14:36,819 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=0e35e57b6f6cbff01d3e782fbf6efd1f, ASSIGN 2024-12-03T23:14:36,821 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=0e35e57b6f6cbff01d3e782fbf6efd1f, ASSIGN; state=OFFLINE, location=5f9d64419733,44899,1733267673202; forceNewPlan=false, retain=false 2024-12-03T23:14:36,823 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f71f90ef2d2f59e885f2ad6ca1a99596, ASSIGN 2024-12-03T23:14:36,825 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=f71f90ef2d2f59e885f2ad6ca1a99596, ASSIGN; state=OFFLINE, location=5f9d64419733,41167,1733267673270; forceNewPlan=false, retain=false 2024-12-03T23:14:36,827 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=6d9981bd2d541debb970a37ac121e7ed, ASSIGN 2024-12-03T23:14:36,828 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=6d9981bd2d541debb970a37ac121e7ed, ASSIGN; state=OFFLINE, location=5f9d64419733,46833,1733267673070; forceNewPlan=false, retain=false 2024-12-03T23:14:36,906 INFO [5f9d64419733:44117 {}] balancer.BaseLoadBalancer(388): Reassigned 26 regions. 26 retained the pre-restart assignment. 2024-12-03T23:14:36,907 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=553e1b6e96399dd43d3b8f7cfad54903, regionState=OPENING, regionLocation=5f9d64419733,46833,1733267673070 2024-12-03T23:14:36,908 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=23a123891f32f583441cd7569a6de1f2, regionState=OPENING, regionLocation=5f9d64419733,46833,1733267673070 2024-12-03T23:14:36,909 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=2b7da3736c6b257b43fe66ea13807bc6, regionState=OPENING, regionLocation=5f9d64419733,46833,1733267673070 2024-12-03T23:14:36,909 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=742c43a49ea3b763b8232c8e8c8966b3, regionState=OPENING, regionLocation=5f9d64419733,46833,1733267673070 2024-12-03T23:14:36,910 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=4e6706440af7674db97bb48a05858115, regionState=OPENING, regionLocation=5f9d64419733,46833,1733267673070 2024-12-03T23:14:36,910 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=384b2e2791a8bf699cb7996e1aa7469c, regionState=OPENING, regionLocation=5f9d64419733,46833,1733267673070 2024-12-03T23:14:36,911 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=256869bf1dc8aa33be79adb19e13d134, regionState=OPENING, regionLocation=5f9d64419733,46833,1733267673070 2024-12-03T23:14:36,911 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=6d9981bd2d541debb970a37ac121e7ed, regionState=OPENING, regionLocation=5f9d64419733,46833,1733267673070 2024-12-03T23:14:36,911 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=553e1b6e96399dd43d3b8f7cfad54903, ASSIGN because future has completed 2024-12-03T23:14:36,912 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=db9924063978f5acb21b073c5d94a7ad, regionState=OPENING, regionLocation=5f9d64419733,44899,1733267673202 2024-12-03T23:14:36,913 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=a0f09ea7cbe3ceec5232d11c21a512a4, regionState=OPENING, regionLocation=5f9d64419733,44899,1733267673202 2024-12-03T23:14:36,913 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 553e1b6e96399dd43d3b8f7cfad54903, server=5f9d64419733,46833,1733267673070}] 2024-12-03T23:14:36,914 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=23a123891f32f583441cd7569a6de1f2, ASSIGN because future has completed 2024-12-03T23:14:36,916 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=2b7da3736c6b257b43fe66ea13807bc6, ASSIGN because future has completed 2024-12-03T23:14:36,918 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=742c43a49ea3b763b8232c8e8c8966b3, ASSIGN because future has completed 2024-12-03T23:14:36,920 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=4e6706440af7674db97bb48a05858115, ASSIGN because future has completed 2024-12-03T23:14:36,925 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4e6706440af7674db97bb48a05858115, server=5f9d64419733,46833,1733267673070}] 2024-12-03T23:14:36,926 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=384b2e2791a8bf699cb7996e1aa7469c, ASSIGN because future has completed 2024-12-03T23:14:36,927 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=256869bf1dc8aa33be79adb19e13d134, ASSIGN because future has completed 2024-12-03T23:14:36,929 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=6d9981bd2d541debb970a37ac121e7ed, ASSIGN because future has completed 2024-12-03T23:14:36,931 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=db9924063978f5acb21b073c5d94a7ad, ASSIGN because future has completed 2024-12-03T23:14:36,931 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure db9924063978f5acb21b073c5d94a7ad, server=5f9d64419733,44899,1733267673202}] 2024-12-03T23:14:36,932 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=a0f09ea7cbe3ceec5232d11c21a512a4, ASSIGN because future has completed 2024-12-03T23:14:36,936 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure a0f09ea7cbe3ceec5232d11c21a512a4, server=5f9d64419733,44899,1733267673202}] 2024-12-03T23:14:36,941 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6d9981bd2d541debb970a37ac121e7ed, server=5f9d64419733,46833,1733267673070}] 2024-12-03T23:14:36,945 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure 256869bf1dc8aa33be79adb19e13d134, server=5f9d64419733,46833,1733267673070}] 2024-12-03T23:14:36,949 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 384b2e2791a8bf699cb7996e1aa7469c, server=5f9d64419733,46833,1733267673070}] 2024-12-03T23:14:36,954 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 742c43a49ea3b763b8232c8e8c8966b3, server=5f9d64419733,46833,1733267673070}] 2024-12-03T23:14:36,959 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2b7da3736c6b257b43fe66ea13807bc6, server=5f9d64419733,46833,1733267673070}] 2024-12-03T23:14:36,963 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 23a123891f32f583441cd7569a6de1f2, server=5f9d64419733,46833,1733267673070}] 2024-12-03T23:14:36,967 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=a6de975926600fad463e13238b2ac8a9, regionState=OPENING, regionLocation=5f9d64419733,44899,1733267673202 2024-12-03T23:14:36,968 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=10beaf4047510d958e4a559194242a57, regionState=OPENING, regionLocation=5f9d64419733,44899,1733267673202 2024-12-03T23:14:36,969 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=f00ca5d778884039f3248393a629dcfc, regionState=OPENING, regionLocation=5f9d64419733,44899,1733267673202 2024-12-03T23:14:36,969 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=5ddac1aa9a6f0af338b83490d6d952c2, regionState=OPENING, regionLocation=5f9d64419733,44899,1733267673202 2024-12-03T23:14:36,970 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=73cd6e72f89a0d14d1d979f76c9c07a4, regionState=OPENING, regionLocation=5f9d64419733,44899,1733267673202 2024-12-03T23:14:36,970 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=f514b27fbd5c6ef451e11852713ea906, regionState=OPENING, regionLocation=5f9d64419733,44899,1733267673202 2024-12-03T23:14:36,971 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=0e35e57b6f6cbff01d3e782fbf6efd1f, regionState=OPENING, regionLocation=5f9d64419733,44899,1733267673202 2024-12-03T23:14:36,971 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=a6de975926600fad463e13238b2ac8a9, ASSIGN because future has completed 2024-12-03T23:14:36,971 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure a6de975926600fad463e13238b2ac8a9, server=5f9d64419733,44899,1733267673202}] 2024-12-03T23:14:36,973 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=10beaf4047510d958e4a559194242a57, ASSIGN because future has completed 2024-12-03T23:14:36,975 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=f00ca5d778884039f3248393a629dcfc, ASSIGN because future has completed 2024-12-03T23:14:36,977 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=42, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure f00ca5d778884039f3248393a629dcfc, server=5f9d64419733,44899,1733267673202}] 2024-12-03T23:14:36,977 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=5ddac1aa9a6f0af338b83490d6d952c2, ASSIGN because future has completed 2024-12-03T23:14:36,980 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=73cd6e72f89a0d14d1d979f76c9c07a4, ASSIGN because future has completed 2024-12-03T23:14:36,982 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=f514b27fbd5c6ef451e11852713ea906, ASSIGN because future has completed 2024-12-03T23:14:36,983 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=0e35e57b6f6cbff01d3e782fbf6efd1f, ASSIGN because future has completed 2024-12-03T23:14:36,983 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0e35e57b6f6cbff01d3e782fbf6efd1f, server=5f9d64419733,44899,1733267673202}] 2024-12-03T23:14:36,987 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=44, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure f514b27fbd5c6ef451e11852713ea906, server=5f9d64419733,44899,1733267673202}] 2024-12-03T23:14:36,992 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 73cd6e72f89a0d14d1d979f76c9c07a4, server=5f9d64419733,44899,1733267673202}] 2024-12-03T23:14:36,996 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=46, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5ddac1aa9a6f0af338b83490d6d952c2, server=5f9d64419733,44899,1733267673202}] 2024-12-03T23:14:37,002 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure 10beaf4047510d958e4a559194242a57, server=5f9d64419733,44899,1733267673202}] 2024-12-03T23:14:37,006 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=e96b12de28d6c3047803f54a3b027459, regionState=OPENING, regionLocation=5f9d64419733,41167,1733267673270 2024-12-03T23:14:37,007 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=e5e4ecc042f874ec5567f7e86548256f, regionState=OPENING, regionLocation=5f9d64419733,41167,1733267673270 2024-12-03T23:14:37,008 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=89a698454f10c9fb6cfc108c35d8415a, regionState=OPENING, regionLocation=5f9d64419733,41167,1733267673270 2024-12-03T23:14:37,008 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=5e6925efcdb64a8de08b4c4bbf3a3d2e, regionState=OPENING, regionLocation=5f9d64419733,41167,1733267673270 2024-12-03T23:14:37,009 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=1d026a7a0318e7deb79fa16b1262ae2c, regionState=OPENING, regionLocation=5f9d64419733,41167,1733267673270 2024-12-03T23:14:37,009 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=2db7cef12ba09a51b0a7204fc81deeec, regionState=OPENING, regionLocation=5f9d64419733,41167,1733267673270 2024-12-03T23:14:37,010 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=212a9963c9cddede26a4eb521cff5254, regionState=OPENING, regionLocation=5f9d64419733,41167,1733267673270 2024-12-03T23:14:37,011 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=55eff67791415c655cef877182b9bbe1, regionState=OPENING, regionLocation=5f9d64419733,41167,1733267673270 2024-12-03T23:14:37,011 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=f71f90ef2d2f59e885f2ad6ca1a99596, regionState=OPENING, regionLocation=5f9d64419733,41167,1733267673270 2024-12-03T23:14:37,011 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=e96b12de28d6c3047803f54a3b027459, ASSIGN because future has completed 2024-12-03T23:14:37,012 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=48, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure e96b12de28d6c3047803f54a3b027459, server=5f9d64419733,41167,1733267673270}] 2024-12-03T23:14:37,013 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=e5e4ecc042f874ec5567f7e86548256f, ASSIGN because future has completed 2024-12-03T23:14:37,017 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=89a698454f10c9fb6cfc108c35d8415a, ASSIGN because future has completed 2024-12-03T23:14:37,019 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure 89a698454f10c9fb6cfc108c35d8415a, server=5f9d64419733,41167,1733267673270}] 2024-12-03T23:14:37,019 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=5e6925efcdb64a8de08b4c4bbf3a3d2e, ASSIGN because future has completed 2024-12-03T23:14:37,021 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=1d026a7a0318e7deb79fa16b1262ae2c, ASSIGN because future has completed 2024-12-03T23:14:37,022 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=2db7cef12ba09a51b0a7204fc81deeec, ASSIGN because future has completed 2024-12-03T23:14:37,024 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=212a9963c9cddede26a4eb521cff5254, ASSIGN because future has completed 2024-12-03T23:14:37,024 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=50, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure 212a9963c9cddede26a4eb521cff5254, server=5f9d64419733,41167,1733267673270}] 2024-12-03T23:14:37,025 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=55eff67791415c655cef877182b9bbe1, ASSIGN because future has completed 2024-12-03T23:14:37,026 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=f71f90ef2d2f59e885f2ad6ca1a99596, ASSIGN because future has completed 2024-12-03T23:14:37,029 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure f71f90ef2d2f59e885f2ad6ca1a99596, server=5f9d64419733,41167,1733267673270}] 2024-12-03T23:14:37,033 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=52, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 55eff67791415c655cef877182b9bbe1, server=5f9d64419733,41167,1733267673270}] 2024-12-03T23:14:37,037 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2db7cef12ba09a51b0a7204fc81deeec, server=5f9d64419733,41167,1733267673270}] 2024-12-03T23:14:37,041 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=54, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1d026a7a0318e7deb79fa16b1262ae2c, server=5f9d64419733,41167,1733267673270}] 2024-12-03T23:14:37,045 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5e6925efcdb64a8de08b4c4bbf3a3d2e, server=5f9d64419733,41167,1733267673270}] 2024-12-03T23:14:37,048 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure e5e4ecc042f874ec5567f7e86548256f, server=5f9d64419733,41167,1733267673270}] 2024-12-03T23:14:37,075 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(132): Open test,jjj,1733267676090.384b2e2791a8bf699cb7996e1aa7469c. 2024-12-03T23:14:37,075 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7752): Opening region: {ENCODED => 384b2e2791a8bf699cb7996e1aa7469c, NAME => 'test,jjj,1733267676090.384b2e2791a8bf699cb7996e1aa7469c.', STARTKEY => 'jjj', ENDKEY => 'kkk'} 2024-12-03T23:14:37,076 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 384b2e2791a8bf699cb7996e1aa7469c 2024-12-03T23:14:37,076 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(898): Instantiated test,jjj,1733267676090.384b2e2791a8bf699cb7996e1aa7469c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T23:14:37,076 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7794): checking encryption for 384b2e2791a8bf699cb7996e1aa7469c 2024-12-03T23:14:37,076 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7797): checking classloading for 384b2e2791a8bf699cb7996e1aa7469c 2024-12-03T23:14:37,078 INFO [StoreOpener-384b2e2791a8bf699cb7996e1aa7469c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 384b2e2791a8bf699cb7996e1aa7469c 2024-12-03T23:14:37,080 INFO [StoreOpener-384b2e2791a8bf699cb7996e1aa7469c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 384b2e2791a8bf699cb7996e1aa7469c columnFamilyName family 2024-12-03T23:14:37,080 DEBUG [StoreOpener-384b2e2791a8bf699cb7996e1aa7469c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:14:37,081 INFO [StoreOpener-384b2e2791a8bf699cb7996e1aa7469c-1 {}] regionserver.HStore(327): Store=384b2e2791a8bf699cb7996e1aa7469c/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T23:14:37,081 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1038): replaying wal for 384b2e2791a8bf699cb7996e1aa7469c 2024-12-03T23:14:37,082 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/default/test/384b2e2791a8bf699cb7996e1aa7469c 2024-12-03T23:14:37,083 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/default/test/384b2e2791a8bf699cb7996e1aa7469c 2024-12-03T23:14:37,083 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1048): stopping wal replay for 384b2e2791a8bf699cb7996e1aa7469c 2024-12-03T23:14:37,083 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1060): Cleaning up temporary data for 384b2e2791a8bf699cb7996e1aa7469c 2024-12-03T23:14:37,085 DEBUG [RSProcedureDispatcher-pool-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-03T23:14:37,086 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1093): writing seq id for 384b2e2791a8bf699cb7996e1aa7469c 2024-12-03T23:14:37,087 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58847, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-03T23:14:37,089 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/default/test/384b2e2791a8bf699cb7996e1aa7469c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T23:14:37,089 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1114): Opened 384b2e2791a8bf699cb7996e1aa7469c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68190029, jitterRate=0.0161106139421463}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T23:14:37,089 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 384b2e2791a8bf699cb7996e1aa7469c 2024-12-03T23:14:37,090 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1006): Region open journal for 384b2e2791a8bf699cb7996e1aa7469c: Running coprocessor pre-open hook at 1733267677076Writing region info on filesystem at 1733267677076Initializing all the Stores at 1733267677078 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733267677078Cleaning up temporary data from old regions at 1733267677083 (+5 ms)Running coprocessor post-open hooks at 1733267677089 (+6 ms)Region opened successfully at 1733267677090 (+1 ms) 2024-12-03T23:14:37,092 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(132): Open test,fff,1733267676090.f514b27fbd5c6ef451e11852713ea906. 2024-12-03T23:14:37,092 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2236): Post open deploy tasks for test,jjj,1733267676090.384b2e2791a8bf699cb7996e1aa7469c., pid=37, masterSystemTime=1733267677068 2024-12-03T23:14:37,092 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7752): Opening region: {ENCODED => f514b27fbd5c6ef451e11852713ea906, NAME => 'test,fff,1733267676090.f514b27fbd5c6ef451e11852713ea906.', STARTKEY => 'fff', ENDKEY => 'ggg'} 2024-12-03T23:14:37,093 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test f514b27fbd5c6ef451e11852713ea906 2024-12-03T23:14:37,093 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(898): Instantiated test,fff,1733267676090.f514b27fbd5c6ef451e11852713ea906.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T23:14:37,093 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7794): checking encryption for f514b27fbd5c6ef451e11852713ea906 2024-12-03T23:14:37,093 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7797): checking classloading for f514b27fbd5c6ef451e11852713ea906 2024-12-03T23:14:37,095 INFO [StoreOpener-f514b27fbd5c6ef451e11852713ea906-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region f514b27fbd5c6ef451e11852713ea906 2024-12-03T23:14:37,096 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2266): Finished post open deploy task for test,jjj,1733267676090.384b2e2791a8bf699cb7996e1aa7469c. 2024-12-03T23:14:37,096 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(153): Opened test,jjj,1733267676090.384b2e2791a8bf699cb7996e1aa7469c. 2024-12-03T23:14:37,096 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(132): Open test,yyy,1733267676090.553e1b6e96399dd43d3b8f7cfad54903. 2024-12-03T23:14:37,096 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7752): Opening region: {ENCODED => 553e1b6e96399dd43d3b8f7cfad54903, NAME => 'test,yyy,1733267676090.553e1b6e96399dd43d3b8f7cfad54903.', STARTKEY => 'yyy', ENDKEY => 'zzz'} 2024-12-03T23:14:37,097 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=384b2e2791a8bf699cb7996e1aa7469c, regionState=OPEN, openSeqNum=2, regionLocation=5f9d64419733,46833,1733267673070 2024-12-03T23:14:37,097 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 553e1b6e96399dd43d3b8f7cfad54903 2024-12-03T23:14:37,097 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(898): Instantiated test,yyy,1733267676090.553e1b6e96399dd43d3b8f7cfad54903.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T23:14:37,097 INFO [StoreOpener-f514b27fbd5c6ef451e11852713ea906-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f514b27fbd5c6ef451e11852713ea906 columnFamilyName family 2024-12-03T23:14:37,097 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7794): checking encryption for 553e1b6e96399dd43d3b8f7cfad54903 2024-12-03T23:14:37,097 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7797): checking classloading for 553e1b6e96399dd43d3b8f7cfad54903 2024-12-03T23:14:37,097 DEBUG [StoreOpener-f514b27fbd5c6ef451e11852713ea906-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:14:37,098 INFO [StoreOpener-f514b27fbd5c6ef451e11852713ea906-1 {}] regionserver.HStore(327): Store=f514b27fbd5c6ef451e11852713ea906/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T23:14:37,099 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1038): replaying wal for f514b27fbd5c6ef451e11852713ea906 2024-12-03T23:14:37,100 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/default/test/f514b27fbd5c6ef451e11852713ea906 2024-12-03T23:14:37,100 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 384b2e2791a8bf699cb7996e1aa7469c, server=5f9d64419733,46833,1733267673070 because future has completed 2024-12-03T23:14:37,101 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/default/test/f514b27fbd5c6ef451e11852713ea906 2024-12-03T23:14:37,101 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1048): stopping wal replay for f514b27fbd5c6ef451e11852713ea906 2024-12-03T23:14:37,102 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1060): Cleaning up temporary data for f514b27fbd5c6ef451e11852713ea906 2024-12-03T23:14:37,104 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1093): writing seq id for f514b27fbd5c6ef451e11852713ea906 2024-12-03T23:14:37,105 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=14 2024-12-03T23:14:37,105 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=14, state=SUCCESS, hasLock=false; OpenRegionProcedure 384b2e2791a8bf699cb7996e1aa7469c, server=5f9d64419733,46833,1733267673070 in 153 msec 2024-12-03T23:14:37,107 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=384b2e2791a8bf699cb7996e1aa7469c, ASSIGN in 363 msec 2024-12-03T23:14:37,108 INFO [StoreOpener-553e1b6e96399dd43d3b8f7cfad54903-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 553e1b6e96399dd43d3b8f7cfad54903 2024-12-03T23:14:37,109 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/default/test/f514b27fbd5c6ef451e11852713ea906/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T23:14:37,110 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1114): Opened f514b27fbd5c6ef451e11852713ea906; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68663879, jitterRate=0.023171529173851013}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T23:14:37,110 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f514b27fbd5c6ef451e11852713ea906 2024-12-03T23:14:37,110 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1006): Region open journal for f514b27fbd5c6ef451e11852713ea906: Running coprocessor pre-open hook at 1733267677093Writing region info on filesystem at 1733267677093Initializing all the Stores at 1733267677094 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733267677094Cleaning up temporary data from old regions at 1733267677102 (+8 ms)Running coprocessor post-open hooks at 1733267677110 (+8 ms)Region opened successfully at 1733267677110 2024-12-03T23:14:37,110 INFO [StoreOpener-553e1b6e96399dd43d3b8f7cfad54903-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 553e1b6e96399dd43d3b8f7cfad54903 columnFamilyName family 2024-12-03T23:14:37,111 DEBUG [StoreOpener-553e1b6e96399dd43d3b8f7cfad54903-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:14:37,112 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2236): Post open deploy tasks for test,fff,1733267676090.f514b27fbd5c6ef451e11852713ea906., pid=44, masterSystemTime=1733267677085 2024-12-03T23:14:37,112 INFO [StoreOpener-553e1b6e96399dd43d3b8f7cfad54903-1 {}] regionserver.HStore(327): Store=553e1b6e96399dd43d3b8f7cfad54903/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T23:14:37,112 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1038): replaying wal for 553e1b6e96399dd43d3b8f7cfad54903 2024-12-03T23:14:37,113 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/default/test/553e1b6e96399dd43d3b8f7cfad54903 2024-12-03T23:14:37,114 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/default/test/553e1b6e96399dd43d3b8f7cfad54903 2024-12-03T23:14:37,115 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1048): stopping wal replay for 553e1b6e96399dd43d3b8f7cfad54903 2024-12-03T23:14:37,115 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1060): Cleaning up temporary data for 553e1b6e96399dd43d3b8f7cfad54903 2024-12-03T23:14:37,115 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2266): Finished post open deploy task for test,fff,1733267676090.f514b27fbd5c6ef451e11852713ea906. 2024-12-03T23:14:37,115 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(153): Opened test,fff,1733267676090.f514b27fbd5c6ef451e11852713ea906. 2024-12-03T23:14:37,115 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(132): Open test,ccc,1733267676090.0e35e57b6f6cbff01d3e782fbf6efd1f. 2024-12-03T23:14:37,116 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7752): Opening region: {ENCODED => 0e35e57b6f6cbff01d3e782fbf6efd1f, NAME => 'test,ccc,1733267676090.0e35e57b6f6cbff01d3e782fbf6efd1f.', STARTKEY => 'ccc', ENDKEY => 'ddd'} 2024-12-03T23:14:37,116 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 0e35e57b6f6cbff01d3e782fbf6efd1f 2024-12-03T23:14:37,116 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=f514b27fbd5c6ef451e11852713ea906, regionState=OPEN, openSeqNum=2, regionLocation=5f9d64419733,44899,1733267673202 2024-12-03T23:14:37,116 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(898): Instantiated test,ccc,1733267676090.0e35e57b6f6cbff01d3e782fbf6efd1f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T23:14:37,116 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7794): checking encryption for 0e35e57b6f6cbff01d3e782fbf6efd1f 2024-12-03T23:14:37,116 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7797): checking classloading for 0e35e57b6f6cbff01d3e782fbf6efd1f 2024-12-03T23:14:37,118 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1093): writing seq id for 553e1b6e96399dd43d3b8f7cfad54903 2024-12-03T23:14:37,118 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44117 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=5f9d64419733,44899,1733267673202, table=test, region=f514b27fbd5c6ef451e11852713ea906. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-03T23:14:37,119 INFO [StoreOpener-0e35e57b6f6cbff01d3e782fbf6efd1f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 0e35e57b6f6cbff01d3e782fbf6efd1f 2024-12-03T23:14:37,120 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=44, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure f514b27fbd5c6ef451e11852713ea906, server=5f9d64419733,44899,1733267673202 because future has completed 2024-12-03T23:14:37,123 INFO [StoreOpener-0e35e57b6f6cbff01d3e782fbf6efd1f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0e35e57b6f6cbff01d3e782fbf6efd1f columnFamilyName family 2024-12-03T23:14:37,123 DEBUG [StoreOpener-0e35e57b6f6cbff01d3e782fbf6efd1f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:14:37,124 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/default/test/553e1b6e96399dd43d3b8f7cfad54903/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T23:14:37,125 INFO [StoreOpener-0e35e57b6f6cbff01d3e782fbf6efd1f-1 {}] regionserver.HStore(327): Store=0e35e57b6f6cbff01d3e782fbf6efd1f/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T23:14:37,125 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1038): replaying wal for 0e35e57b6f6cbff01d3e782fbf6efd1f 2024-12-03T23:14:37,125 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=44, resume processing ppid=10 2024-12-03T23:14:37,126 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure f514b27fbd5c6ef451e11852713ea906, server=5f9d64419733,44899,1733267673202 in 135 msec 2024-12-03T23:14:37,126 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1114): Opened 553e1b6e96399dd43d3b8f7cfad54903; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62184075, jitterRate=-0.0733850747346878}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T23:14:37,126 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 553e1b6e96399dd43d3b8f7cfad54903 2024-12-03T23:14:37,126 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/default/test/0e35e57b6f6cbff01d3e782fbf6efd1f 2024-12-03T23:14:37,126 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1006): Region open journal for 553e1b6e96399dd43d3b8f7cfad54903: Running coprocessor pre-open hook at 1733267677097Writing region info on filesystem at 1733267677097Initializing all the Stores at 1733267677099 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733267677099Cleaning up temporary data from old regions at 1733267677115 (+16 ms)Running coprocessor post-open hooks at 1733267677126 (+11 ms)Region opened successfully at 1733267677126 2024-12-03T23:14:37,127 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/default/test/0e35e57b6f6cbff01d3e782fbf6efd1f 2024-12-03T23:14:37,127 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2236): Post open deploy tasks for test,yyy,1733267676090.553e1b6e96399dd43d3b8f7cfad54903., pid=31, masterSystemTime=1733267677068 2024-12-03T23:14:37,128 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1048): stopping wal replay for 0e35e57b6f6cbff01d3e782fbf6efd1f 2024-12-03T23:14:37,128 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1060): Cleaning up temporary data for 0e35e57b6f6cbff01d3e782fbf6efd1f 2024-12-03T23:14:37,130 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=f514b27fbd5c6ef451e11852713ea906, ASSIGN in 384 msec 2024-12-03T23:14:37,130 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2266): Finished post open deploy task for test,yyy,1733267676090.553e1b6e96399dd43d3b8f7cfad54903. 2024-12-03T23:14:37,130 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(153): Opened test,yyy,1733267676090.553e1b6e96399dd43d3b8f7cfad54903. 2024-12-03T23:14:37,131 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(132): Open test,hhh,1733267676090.256869bf1dc8aa33be79adb19e13d134. 2024-12-03T23:14:37,131 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1093): writing seq id for 0e35e57b6f6cbff01d3e782fbf6efd1f 2024-12-03T23:14:37,131 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7752): Opening region: {ENCODED => 256869bf1dc8aa33be79adb19e13d134, NAME => 'test,hhh,1733267676090.256869bf1dc8aa33be79adb19e13d134.', STARTKEY => 'hhh', ENDKEY => 'iii'} 2024-12-03T23:14:37,131 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=553e1b6e96399dd43d3b8f7cfad54903, regionState=OPEN, openSeqNum=2, regionLocation=5f9d64419733,46833,1733267673070 2024-12-03T23:14:37,131 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 256869bf1dc8aa33be79adb19e13d134 2024-12-03T23:14:37,132 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(898): Instantiated test,hhh,1733267676090.256869bf1dc8aa33be79adb19e13d134.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T23:14:37,132 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7794): checking encryption for 256869bf1dc8aa33be79adb19e13d134 2024-12-03T23:14:37,132 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7797): checking classloading for 256869bf1dc8aa33be79adb19e13d134 2024-12-03T23:14:37,134 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 553e1b6e96399dd43d3b8f7cfad54903, server=5f9d64419733,46833,1733267673070 because future has completed 2024-12-03T23:14:37,135 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/default/test/0e35e57b6f6cbff01d3e782fbf6efd1f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T23:14:37,136 INFO [StoreOpener-256869bf1dc8aa33be79adb19e13d134-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 256869bf1dc8aa33be79adb19e13d134 2024-12-03T23:14:37,136 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1114): Opened 0e35e57b6f6cbff01d3e782fbf6efd1f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63054607, jitterRate=-0.06041313707828522}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T23:14:37,136 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0e35e57b6f6cbff01d3e782fbf6efd1f 2024-12-03T23:14:37,137 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1006): Region open journal for 0e35e57b6f6cbff01d3e782fbf6efd1f: Running coprocessor pre-open hook at 1733267677117Writing region info on filesystem at 1733267677117Initializing all the Stores at 1733267677118 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733267677118Cleaning up temporary data from old regions at 1733267677128 (+10 ms)Running coprocessor post-open hooks at 1733267677137 (+9 ms)Region opened successfully at 1733267677137 2024-12-03T23:14:37,138 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ccc,1733267676090.0e35e57b6f6cbff01d3e782fbf6efd1f., pid=43, masterSystemTime=1733267677085 2024-12-03T23:14:37,140 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=29 2024-12-03T23:14:37,140 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure 553e1b6e96399dd43d3b8f7cfad54903, server=5f9d64419733,46833,1733267673070 in 223 msec 2024-12-03T23:14:37,141 INFO [StoreOpener-256869bf1dc8aa33be79adb19e13d134-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 256869bf1dc8aa33be79adb19e13d134 columnFamilyName family 2024-12-03T23:14:37,141 DEBUG [StoreOpener-256869bf1dc8aa33be79adb19e13d134-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:14:37,142 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ccc,1733267676090.0e35e57b6f6cbff01d3e782fbf6efd1f. 2024-12-03T23:14:37,142 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(153): Opened test,ccc,1733267676090.0e35e57b6f6cbff01d3e782fbf6efd1f. 2024-12-03T23:14:37,142 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(132): Open test,ggg,1733267676090.73cd6e72f89a0d14d1d979f76c9c07a4. 2024-12-03T23:14:37,143 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7752): Opening region: {ENCODED => 73cd6e72f89a0d14d1d979f76c9c07a4, NAME => 'test,ggg,1733267676090.73cd6e72f89a0d14d1d979f76c9c07a4.', STARTKEY => 'ggg', ENDKEY => 'hhh'} 2024-12-03T23:14:37,143 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=553e1b6e96399dd43d3b8f7cfad54903, ASSIGN in 398 msec 2024-12-03T23:14:37,143 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 73cd6e72f89a0d14d1d979f76c9c07a4 2024-12-03T23:14:37,143 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(898): Instantiated test,ggg,1733267676090.73cd6e72f89a0d14d1d979f76c9c07a4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T23:14:37,143 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7794): checking encryption for 73cd6e72f89a0d14d1d979f76c9c07a4 2024-12-03T23:14:37,143 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7797): checking classloading for 73cd6e72f89a0d14d1d979f76c9c07a4 2024-12-03T23:14:37,144 INFO [StoreOpener-256869bf1dc8aa33be79adb19e13d134-1 {}] regionserver.HStore(327): Store=256869bf1dc8aa33be79adb19e13d134/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T23:14:37,144 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1038): replaying wal for 256869bf1dc8aa33be79adb19e13d134 2024-12-03T23:14:37,144 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=0e35e57b6f6cbff01d3e782fbf6efd1f, regionState=OPEN, openSeqNum=2, regionLocation=5f9d64419733,44899,1733267673202 2024-12-03T23:14:37,145 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/default/test/256869bf1dc8aa33be79adb19e13d134 2024-12-03T23:14:37,146 INFO [StoreOpener-73cd6e72f89a0d14d1d979f76c9c07a4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 73cd6e72f89a0d14d1d979f76c9c07a4 2024-12-03T23:14:37,146 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/default/test/256869bf1dc8aa33be79adb19e13d134 2024-12-03T23:14:37,147 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1048): stopping wal replay for 256869bf1dc8aa33be79adb19e13d134 2024-12-03T23:14:37,147 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1060): Cleaning up temporary data for 256869bf1dc8aa33be79adb19e13d134 2024-12-03T23:14:37,148 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=43, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0e35e57b6f6cbff01d3e782fbf6efd1f, server=5f9d64419733,44899,1733267673202 because future has completed 2024-12-03T23:14:37,148 INFO [StoreOpener-73cd6e72f89a0d14d1d979f76c9c07a4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 73cd6e72f89a0d14d1d979f76c9c07a4 columnFamilyName family 2024-12-03T23:14:37,149 DEBUG [StoreOpener-73cd6e72f89a0d14d1d979f76c9c07a4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:14:37,149 INFO [StoreOpener-73cd6e72f89a0d14d1d979f76c9c07a4-1 {}] regionserver.HStore(327): Store=73cd6e72f89a0d14d1d979f76c9c07a4/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T23:14:37,150 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1038): replaying wal for 73cd6e72f89a0d14d1d979f76c9c07a4 2024-12-03T23:14:37,151 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1093): writing seq id for 256869bf1dc8aa33be79adb19e13d134 2024-12-03T23:14:37,151 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/default/test/73cd6e72f89a0d14d1d979f76c9c07a4 2024-12-03T23:14:37,152 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/default/test/73cd6e72f89a0d14d1d979f76c9c07a4 2024-12-03T23:14:37,152 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1048): stopping wal replay for 73cd6e72f89a0d14d1d979f76c9c07a4 2024-12-03T23:14:37,152 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1060): Cleaning up temporary data for 73cd6e72f89a0d14d1d979f76c9c07a4 2024-12-03T23:14:37,155 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=7 2024-12-03T23:14:37,155 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/default/test/256869bf1dc8aa33be79adb19e13d134/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T23:14:37,155 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 0e35e57b6f6cbff01d3e782fbf6efd1f, server=5f9d64419733,44899,1733267673202 in 167 msec 2024-12-03T23:14:37,156 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1114): Opened 256869bf1dc8aa33be79adb19e13d134; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71455621, jitterRate=0.06477172672748566}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T23:14:37,156 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 256869bf1dc8aa33be79adb19e13d134 2024-12-03T23:14:37,156 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1093): writing seq id for 73cd6e72f89a0d14d1d979f76c9c07a4 2024-12-03T23:14:37,156 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1006): Region open journal for 256869bf1dc8aa33be79adb19e13d134: Running coprocessor pre-open hook at 1733267677132Writing region info on filesystem at 1733267677132Initializing all the Stores at 1733267677134 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733267677134Cleaning up temporary data from old regions at 1733267677147 (+13 ms)Running coprocessor post-open hooks at 1733267677156 (+9 ms)Region opened successfully at 1733267677156 2024-12-03T23:14:37,158 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2236): Post open deploy tasks for test,hhh,1733267676090.256869bf1dc8aa33be79adb19e13d134., pid=36, masterSystemTime=1733267677068 2024-12-03T23:14:37,160 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=0e35e57b6f6cbff01d3e782fbf6efd1f, ASSIGN in 413 msec 2024-12-03T23:14:37,161 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2266): Finished post open deploy task for test,hhh,1733267676090.256869bf1dc8aa33be79adb19e13d134. 2024-12-03T23:14:37,161 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(153): Opened test,hhh,1733267676090.256869bf1dc8aa33be79adb19e13d134. 2024-12-03T23:14:37,161 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(132): Open test,vvv,1733267676090.23a123891f32f583441cd7569a6de1f2. 2024-12-03T23:14:37,162 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7752): Opening region: {ENCODED => 23a123891f32f583441cd7569a6de1f2, NAME => 'test,vvv,1733267676090.23a123891f32f583441cd7569a6de1f2.', STARTKEY => 'vvv', ENDKEY => 'www'} 2024-12-03T23:14:37,162 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=256869bf1dc8aa33be79adb19e13d134, regionState=OPEN, openSeqNum=2, regionLocation=5f9d64419733,46833,1733267673070 2024-12-03T23:14:37,162 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 23a123891f32f583441cd7569a6de1f2 2024-12-03T23:14:37,162 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(898): Instantiated test,vvv,1733267676090.23a123891f32f583441cd7569a6de1f2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T23:14:37,162 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7794): checking encryption for 23a123891f32f583441cd7569a6de1f2 2024-12-03T23:14:37,162 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7797): checking classloading for 23a123891f32f583441cd7569a6de1f2 2024-12-03T23:14:37,164 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/default/test/73cd6e72f89a0d14d1d979f76c9c07a4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T23:14:37,165 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1114): Opened 73cd6e72f89a0d14d1d979f76c9c07a4; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63176066, jitterRate=-0.058603256940841675}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T23:14:37,165 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 73cd6e72f89a0d14d1d979f76c9c07a4 2024-12-03T23:14:37,165 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1006): Region open journal for 73cd6e72f89a0d14d1d979f76c9c07a4: Running coprocessor pre-open hook at 1733267677144Writing region info on filesystem at 1733267677144Initializing all the Stores at 1733267677145 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733267677145Cleaning up temporary data from old regions at 1733267677153 (+8 ms)Running coprocessor post-open hooks at 1733267677165 (+12 ms)Region opened successfully at 1733267677165 2024-12-03T23:14:37,165 INFO [StoreOpener-23a123891f32f583441cd7569a6de1f2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 23a123891f32f583441cd7569a6de1f2 2024-12-03T23:14:37,165 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure 256869bf1dc8aa33be79adb19e13d134, server=5f9d64419733,46833,1733267673070 because future has completed 2024-12-03T23:14:37,166 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-03T23:14:37,167 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ggg,1733267676090.73cd6e72f89a0d14d1d979f76c9c07a4., pid=45, masterSystemTime=1733267677085 2024-12-03T23:14:37,168 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55933, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-03T23:14:37,169 INFO [StoreOpener-23a123891f32f583441cd7569a6de1f2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 23a123891f32f583441cd7569a6de1f2 columnFamilyName family 2024-12-03T23:14:37,169 DEBUG [StoreOpener-23a123891f32f583441cd7569a6de1f2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:14:37,170 INFO [StoreOpener-23a123891f32f583441cd7569a6de1f2-1 {}] regionserver.HStore(327): Store=23a123891f32f583441cd7569a6de1f2/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T23:14:37,170 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1038): replaying wal for 23a123891f32f583441cd7569a6de1f2 2024-12-03T23:14:37,171 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/default/test/23a123891f32f583441cd7569a6de1f2 2024-12-03T23:14:37,172 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/default/test/23a123891f32f583441cd7569a6de1f2 2024-12-03T23:14:37,172 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ggg,1733267676090.73cd6e72f89a0d14d1d979f76c9c07a4. 2024-12-03T23:14:37,173 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(132): Open test,bbb,1733267676090.f71f90ef2d2f59e885f2ad6ca1a99596. 2024-12-03T23:14:37,173 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(153): Opened test,ggg,1733267676090.73cd6e72f89a0d14d1d979f76c9c07a4. 2024-12-03T23:14:37,173 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1048): stopping wal replay for 23a123891f32f583441cd7569a6de1f2 2024-12-03T23:14:37,173 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1060): Cleaning up temporary data for 23a123891f32f583441cd7569a6de1f2 2024-12-03T23:14:37,174 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(132): Open test,kkk,1733267676090.5ddac1aa9a6f0af338b83490d6d952c2. 2024-12-03T23:14:37,174 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7752): Opening region: {ENCODED => 5ddac1aa9a6f0af338b83490d6d952c2, NAME => 'test,kkk,1733267676090.5ddac1aa9a6f0af338b83490d6d952c2.', STARTKEY => 'kkk', ENDKEY => 'lll'} 2024-12-03T23:14:37,174 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7752): Opening region: {ENCODED => f71f90ef2d2f59e885f2ad6ca1a99596, NAME => 'test,bbb,1733267676090.f71f90ef2d2f59e885f2ad6ca1a99596.', STARTKEY => 'bbb', ENDKEY => 'ccc'} 2024-12-03T23:14:37,174 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=12 2024-12-03T23:14:37,174 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=12, state=SUCCESS, hasLock=false; OpenRegionProcedure 256869bf1dc8aa33be79adb19e13d134, server=5f9d64419733,46833,1733267673070 in 223 msec 2024-12-03T23:14:37,174 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 5ddac1aa9a6f0af338b83490d6d952c2 2024-12-03T23:14:37,174 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test f71f90ef2d2f59e885f2ad6ca1a99596 2024-12-03T23:14:37,174 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(898): Instantiated test,kkk,1733267676090.5ddac1aa9a6f0af338b83490d6d952c2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T23:14:37,174 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(898): Instantiated test,bbb,1733267676090.f71f90ef2d2f59e885f2ad6ca1a99596.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T23:14:37,175 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7794): checking encryption for 5ddac1aa9a6f0af338b83490d6d952c2 2024-12-03T23:14:37,175 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7794): checking encryption for f71f90ef2d2f59e885f2ad6ca1a99596 2024-12-03T23:14:37,175 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7797): checking classloading for 5ddac1aa9a6f0af338b83490d6d952c2 2024-12-03T23:14:37,175 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7797): checking classloading for f71f90ef2d2f59e885f2ad6ca1a99596 2024-12-03T23:14:37,177 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=256869bf1dc8aa33be79adb19e13d134, ASSIGN in 432 msec 2024-12-03T23:14:37,178 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1093): writing seq id for 23a123891f32f583441cd7569a6de1f2 2024-12-03T23:14:37,178 INFO [StoreOpener-f71f90ef2d2f59e885f2ad6ca1a99596-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region f71f90ef2d2f59e885f2ad6ca1a99596 2024-12-03T23:14:37,178 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=73cd6e72f89a0d14d1d979f76c9c07a4, regionState=OPEN, openSeqNum=2, regionLocation=5f9d64419733,44899,1733267673202 2024-12-03T23:14:37,179 INFO [StoreOpener-5ddac1aa9a6f0af338b83490d6d952c2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 5ddac1aa9a6f0af338b83490d6d952c2 2024-12-03T23:14:37,180 INFO [StoreOpener-f71f90ef2d2f59e885f2ad6ca1a99596-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f71f90ef2d2f59e885f2ad6ca1a99596 columnFamilyName family 2024-12-03T23:14:37,180 DEBUG [StoreOpener-f71f90ef2d2f59e885f2ad6ca1a99596-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:14:37,182 INFO [StoreOpener-f71f90ef2d2f59e885f2ad6ca1a99596-1 {}] regionserver.HStore(327): Store=f71f90ef2d2f59e885f2ad6ca1a99596/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T23:14:37,182 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1038): replaying wal for f71f90ef2d2f59e885f2ad6ca1a99596 2024-12-03T23:14:37,183 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=45, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 73cd6e72f89a0d14d1d979f76c9c07a4, server=5f9d64419733,44899,1733267673202 because future has completed 2024-12-03T23:14:37,183 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/default/test/f71f90ef2d2f59e885f2ad6ca1a99596 2024-12-03T23:14:37,183 INFO [StoreOpener-5ddac1aa9a6f0af338b83490d6d952c2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5ddac1aa9a6f0af338b83490d6d952c2 columnFamilyName family 2024-12-03T23:14:37,183 DEBUG [StoreOpener-5ddac1aa9a6f0af338b83490d6d952c2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:14:37,183 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/default/test/f71f90ef2d2f59e885f2ad6ca1a99596 2024-12-03T23:14:37,184 INFO [StoreOpener-5ddac1aa9a6f0af338b83490d6d952c2-1 {}] regionserver.HStore(327): Store=5ddac1aa9a6f0af338b83490d6d952c2/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T23:14:37,184 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1048): stopping wal replay for f71f90ef2d2f59e885f2ad6ca1a99596 2024-12-03T23:14:37,184 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1038): replaying wal for 5ddac1aa9a6f0af338b83490d6d952c2 2024-12-03T23:14:37,184 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1060): Cleaning up temporary data for f71f90ef2d2f59e885f2ad6ca1a99596 2024-12-03T23:14:37,186 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/default/test/5ddac1aa9a6f0af338b83490d6d952c2 2024-12-03T23:14:37,187 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/default/test/5ddac1aa9a6f0af338b83490d6d952c2 2024-12-03T23:14:37,187 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1093): writing seq id for f71f90ef2d2f59e885f2ad6ca1a99596 2024-12-03T23:14:37,187 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1048): stopping wal replay for 5ddac1aa9a6f0af338b83490d6d952c2 2024-12-03T23:14:37,187 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1060): Cleaning up temporary data for 5ddac1aa9a6f0af338b83490d6d952c2 2024-12-03T23:14:37,188 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=11 2024-12-03T23:14:37,188 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure 73cd6e72f89a0d14d1d979f76c9c07a4, server=5f9d64419733,44899,1733267673202 in 193 msec 2024-12-03T23:14:37,190 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1093): writing seq id for 5ddac1aa9a6f0af338b83490d6d952c2 2024-12-03T23:14:37,191 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=73cd6e72f89a0d14d1d979f76c9c07a4, ASSIGN in 446 msec 2024-12-03T23:14:37,193 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/default/test/23a123891f32f583441cd7569a6de1f2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T23:14:37,194 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1114): Opened 23a123891f32f583441cd7569a6de1f2; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65027485, jitterRate=-0.03101496398448944}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T23:14:37,194 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 23a123891f32f583441cd7569a6de1f2 2024-12-03T23:14:37,194 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1006): Region open journal for 23a123891f32f583441cd7569a6de1f2: Running coprocessor pre-open hook at 1733267677163Writing region info on filesystem at 1733267677163Initializing all the Stores at 1733267677164 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733267677165 (+1 ms)Cleaning up temporary data from old regions at 1733267677173 (+8 ms)Running coprocessor post-open hooks at 1733267677194 (+21 ms)Region opened successfully at 1733267677194 2024-12-03T23:14:37,195 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2236): Post open deploy tasks for test,vvv,1733267676090.23a123891f32f583441cd7569a6de1f2., pid=40, masterSystemTime=1733267677068 2024-12-03T23:14:37,198 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/default/test/f71f90ef2d2f59e885f2ad6ca1a99596/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T23:14:37,198 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2266): Finished post open deploy task for test,vvv,1733267676090.23a123891f32f583441cd7569a6de1f2. 2024-12-03T23:14:37,198 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(153): Opened test,vvv,1733267676090.23a123891f32f583441cd7569a6de1f2. 2024-12-03T23:14:37,199 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(132): Open test,ppp,1733267676090.742c43a49ea3b763b8232c8e8c8966b3. 2024-12-03T23:14:37,199 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7752): Opening region: {ENCODED => 742c43a49ea3b763b8232c8e8c8966b3, NAME => 'test,ppp,1733267676090.742c43a49ea3b763b8232c8e8c8966b3.', STARTKEY => 'ppp', ENDKEY => 'qqq'} 2024-12-03T23:14:37,199 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1114): Opened f71f90ef2d2f59e885f2ad6ca1a99596; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72295613, jitterRate=0.07728858292102814}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T23:14:37,199 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f71f90ef2d2f59e885f2ad6ca1a99596 2024-12-03T23:14:37,200 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1006): Region open journal for f71f90ef2d2f59e885f2ad6ca1a99596: Running coprocessor pre-open hook at 1733267677175Writing region info on filesystem at 1733267677175Initializing all the Stores at 1733267677177 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733267677177Cleaning up temporary data from old regions at 1733267677184 (+7 ms)Running coprocessor post-open hooks at 1733267677199 (+15 ms)Region opened successfully at 1733267677200 (+1 ms) 2024-12-03T23:14:37,200 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 742c43a49ea3b763b8232c8e8c8966b3 2024-12-03T23:14:37,200 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(898): Instantiated test,ppp,1733267676090.742c43a49ea3b763b8232c8e8c8966b3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T23:14:37,200 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7794): checking encryption for 742c43a49ea3b763b8232c8e8c8966b3 2024-12-03T23:14:37,200 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7797): checking classloading for 742c43a49ea3b763b8232c8e8c8966b3 2024-12-03T23:14:37,200 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=23a123891f32f583441cd7569a6de1f2, regionState=OPEN, openSeqNum=2, regionLocation=5f9d64419733,46833,1733267673070 2024-12-03T23:14:37,201 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2236): Post open deploy tasks for test,bbb,1733267676090.f71f90ef2d2f59e885f2ad6ca1a99596., pid=51, masterSystemTime=1733267677166 2024-12-03T23:14:37,205 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/default/test/5ddac1aa9a6f0af338b83490d6d952c2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T23:14:37,206 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1114): Opened 5ddac1aa9a6f0af338b83490d6d952c2; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71968754, jitterRate=0.07241800427436829}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T23:14:37,206 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5ddac1aa9a6f0af338b83490d6d952c2 2024-12-03T23:14:37,206 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1006): Region open journal for 5ddac1aa9a6f0af338b83490d6d952c2: Running coprocessor pre-open hook at 1733267677175Writing region info on filesystem at 1733267677175Initializing all the Stores at 1733267677178 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733267677178Cleaning up temporary data from old regions at 1733267677187 (+9 ms)Running coprocessor post-open hooks at 1733267677206 (+19 ms)Region opened successfully at 1733267677206 2024-12-03T23:14:37,207 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2236): Post open deploy tasks for test,kkk,1733267676090.5ddac1aa9a6f0af338b83490d6d952c2., pid=46, masterSystemTime=1733267677085 2024-12-03T23:14:37,208 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=40, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 23a123891f32f583441cd7569a6de1f2, server=5f9d64419733,46833,1733267673070 because future has completed 2024-12-03T23:14:37,212 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2266): Finished post open deploy task for test,bbb,1733267676090.f71f90ef2d2f59e885f2ad6ca1a99596. 2024-12-03T23:14:37,212 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(153): Opened test,bbb,1733267676090.f71f90ef2d2f59e885f2ad6ca1a99596. 2024-12-03T23:14:37,212 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(132): Open test,rrr,1733267676090.89a698454f10c9fb6cfc108c35d8415a. 2024-12-03T23:14:37,212 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7752): Opening region: {ENCODED => 89a698454f10c9fb6cfc108c35d8415a, NAME => 'test,rrr,1733267676090.89a698454f10c9fb6cfc108c35d8415a.', STARTKEY => 'rrr', ENDKEY => 'sss'} 2024-12-03T23:14:37,213 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 89a698454f10c9fb6cfc108c35d8415a 2024-12-03T23:14:37,213 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(898): Instantiated test,rrr,1733267676090.89a698454f10c9fb6cfc108c35d8415a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T23:14:37,213 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7794): checking encryption for 89a698454f10c9fb6cfc108c35d8415a 2024-12-03T23:14:37,213 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7797): checking classloading for 89a698454f10c9fb6cfc108c35d8415a 2024-12-03T23:14:37,216 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2266): Finished post open deploy task for test,kkk,1733267676090.5ddac1aa9a6f0af338b83490d6d952c2. 2024-12-03T23:14:37,216 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(153): Opened test,kkk,1733267676090.5ddac1aa9a6f0af338b83490d6d952c2. 2024-12-03T23:14:37,217 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(132): Open test,ttt,1733267676090.a6de975926600fad463e13238b2ac8a9. 2024-12-03T23:14:37,217 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7752): Opening region: {ENCODED => a6de975926600fad463e13238b2ac8a9, NAME => 'test,ttt,1733267676090.a6de975926600fad463e13238b2ac8a9.', STARTKEY => 'ttt', ENDKEY => 'uuu'} 2024-12-03T23:14:37,217 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test a6de975926600fad463e13238b2ac8a9 2024-12-03T23:14:37,217 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(898): Instantiated test,ttt,1733267676090.a6de975926600fad463e13238b2ac8a9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T23:14:37,218 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7794): checking encryption for a6de975926600fad463e13238b2ac8a9 2024-12-03T23:14:37,218 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7797): checking classloading for a6de975926600fad463e13238b2ac8a9 2024-12-03T23:14:37,221 INFO [StoreOpener-742c43a49ea3b763b8232c8e8c8966b3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 742c43a49ea3b763b8232c8e8c8966b3 2024-12-03T23:14:37,222 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=26 2024-12-03T23:14:37,222 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=26, state=SUCCESS, hasLock=false; OpenRegionProcedure 23a123891f32f583441cd7569a6de1f2, server=5f9d64419733,46833,1733267673070 in 247 msec 2024-12-03T23:14:37,224 INFO [StoreOpener-742c43a49ea3b763b8232c8e8c8966b3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 742c43a49ea3b763b8232c8e8c8966b3 columnFamilyName family 2024-12-03T23:14:37,224 DEBUG [StoreOpener-742c43a49ea3b763b8232c8e8c8966b3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:14:37,225 INFO [StoreOpener-742c43a49ea3b763b8232c8e8c8966b3-1 {}] regionserver.HStore(327): Store=742c43a49ea3b763b8232c8e8c8966b3/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T23:14:37,225 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1038): replaying wal for 742c43a49ea3b763b8232c8e8c8966b3 2024-12-03T23:14:37,226 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/default/test/742c43a49ea3b763b8232c8e8c8966b3 2024-12-03T23:14:37,227 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=23a123891f32f583441cd7569a6de1f2, ASSIGN in 480 msec 2024-12-03T23:14:37,227 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/default/test/742c43a49ea3b763b8232c8e8c8966b3 2024-12-03T23:14:37,228 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1048): stopping wal replay for 742c43a49ea3b763b8232c8e8c8966b3 2024-12-03T23:14:37,228 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1060): Cleaning up temporary data for 742c43a49ea3b763b8232c8e8c8966b3 2024-12-03T23:14:37,229 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=5ddac1aa9a6f0af338b83490d6d952c2, regionState=OPEN, openSeqNum=2, regionLocation=5f9d64419733,44899,1733267673202 2024-12-03T23:14:37,229 INFO [StoreOpener-89a698454f10c9fb6cfc108c35d8415a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 89a698454f10c9fb6cfc108c35d8415a 2024-12-03T23:14:37,230 INFO [StoreOpener-a6de975926600fad463e13238b2ac8a9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region a6de975926600fad463e13238b2ac8a9 2024-12-03T23:14:37,232 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1093): writing seq id for 742c43a49ea3b763b8232c8e8c8966b3 2024-12-03T23:14:37,232 INFO [StoreOpener-89a698454f10c9fb6cfc108c35d8415a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 89a698454f10c9fb6cfc108c35d8415a columnFamilyName family 2024-12-03T23:14:37,232 DEBUG [StoreOpener-89a698454f10c9fb6cfc108c35d8415a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:14:37,232 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=46, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5ddac1aa9a6f0af338b83490d6d952c2, server=5f9d64419733,44899,1733267673202 because future has completed 2024-12-03T23:14:37,233 INFO [StoreOpener-89a698454f10c9fb6cfc108c35d8415a-1 {}] regionserver.HStore(327): Store=89a698454f10c9fb6cfc108c35d8415a/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T23:14:37,233 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1038): replaying wal for 89a698454f10c9fb6cfc108c35d8415a 2024-12-03T23:14:37,235 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/default/test/89a698454f10c9fb6cfc108c35d8415a 2024-12-03T23:14:37,235 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=f71f90ef2d2f59e885f2ad6ca1a99596, regionState=OPEN, openSeqNum=2, regionLocation=5f9d64419733,41167,1733267673270 2024-12-03T23:14:37,235 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/default/test/89a698454f10c9fb6cfc108c35d8415a 2024-12-03T23:14:37,236 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1048): stopping wal replay for 89a698454f10c9fb6cfc108c35d8415a 2024-12-03T23:14:37,236 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1060): Cleaning up temporary data for 89a698454f10c9fb6cfc108c35d8415a 2024-12-03T23:14:37,236 INFO [StoreOpener-a6de975926600fad463e13238b2ac8a9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a6de975926600fad463e13238b2ac8a9 columnFamilyName family 2024-12-03T23:14:37,236 DEBUG [StoreOpener-a6de975926600fad463e13238b2ac8a9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:14:37,237 INFO [StoreOpener-a6de975926600fad463e13238b2ac8a9-1 {}] regionserver.HStore(327): Store=a6de975926600fad463e13238b2ac8a9/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T23:14:37,237 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1038): replaying wal for a6de975926600fad463e13238b2ac8a9 2024-12-03T23:14:37,239 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=51, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure f71f90ef2d2f59e885f2ad6ca1a99596, server=5f9d64419733,41167,1733267673270 because future has completed 2024-12-03T23:14:37,239 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/default/test/a6de975926600fad463e13238b2ac8a9 2024-12-03T23:14:37,240 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1093): writing seq id for 89a698454f10c9fb6cfc108c35d8415a 2024-12-03T23:14:37,240 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/default/test/a6de975926600fad463e13238b2ac8a9 2024-12-03T23:14:37,241 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1048): stopping wal replay for a6de975926600fad463e13238b2ac8a9 2024-12-03T23:14:37,241 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1060): Cleaning up temporary data for a6de975926600fad463e13238b2ac8a9 2024-12-03T23:14:37,244 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1093): writing seq id for a6de975926600fad463e13238b2ac8a9 2024-12-03T23:14:37,247 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=46, resume processing ppid=15 2024-12-03T23:14:37,248 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, ppid=15, state=SUCCESS, hasLock=false; OpenRegionProcedure 5ddac1aa9a6f0af338b83490d6d952c2, server=5f9d64419733,44899,1733267673202 in 245 msec 2024-12-03T23:14:37,250 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/default/test/742c43a49ea3b763b8232c8e8c8966b3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T23:14:37,251 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1114): Opened 742c43a49ea3b763b8232c8e8c8966b3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70045468, jitterRate=0.0437588095664978}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T23:14:37,251 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 742c43a49ea3b763b8232c8e8c8966b3 2024-12-03T23:14:37,251 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1006): Region open journal for 742c43a49ea3b763b8232c8e8c8966b3: Running coprocessor pre-open hook at 1733267677200Writing region info on filesystem at 1733267677200Initializing all the Stores at 1733267677207 (+7 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733267677207Cleaning up temporary data from old regions at 1733267677228 (+21 ms)Running coprocessor post-open hooks at 1733267677251 (+23 ms)Region opened successfully at 1733267677251 2024-12-03T23:14:37,253 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ppp,1733267676090.742c43a49ea3b763b8232c8e8c8966b3., pid=38, masterSystemTime=1733267677068 2024-12-03T23:14:37,253 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=5ddac1aa9a6f0af338b83490d6d952c2, ASSIGN in 506 msec 2024-12-03T23:14:37,257 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ppp,1733267676090.742c43a49ea3b763b8232c8e8c8966b3. 2024-12-03T23:14:37,257 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(153): Opened test,ppp,1733267676090.742c43a49ea3b763b8232c8e8c8966b3. 2024-12-03T23:14:37,257 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(132): Open test,mmm,1733267676090.4e6706440af7674db97bb48a05858115. 2024-12-03T23:14:37,257 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7752): Opening region: {ENCODED => 4e6706440af7674db97bb48a05858115, NAME => 'test,mmm,1733267676090.4e6706440af7674db97bb48a05858115.', STARTKEY => 'mmm', ENDKEY => 'nnn'} 2024-12-03T23:14:37,258 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 4e6706440af7674db97bb48a05858115 2024-12-03T23:14:37,258 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(898): Instantiated test,mmm,1733267676090.4e6706440af7674db97bb48a05858115.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T23:14:37,258 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7794): checking encryption for 4e6706440af7674db97bb48a05858115 2024-12-03T23:14:37,258 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7797): checking classloading for 4e6706440af7674db97bb48a05858115 2024-12-03T23:14:37,259 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/default/test/a6de975926600fad463e13238b2ac8a9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T23:14:37,260 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1114): Opened a6de975926600fad463e13238b2ac8a9; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63486174, jitterRate=-0.053982287645339966}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T23:14:37,260 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a6de975926600fad463e13238b2ac8a9 2024-12-03T23:14:37,260 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1006): Region open journal for a6de975926600fad463e13238b2ac8a9: Running coprocessor pre-open hook at 1733267677218Writing region info on filesystem at 1733267677218Initializing all the Stores at 1733267677222 (+4 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733267677222Cleaning up temporary data from old regions at 1733267677241 (+19 ms)Running coprocessor post-open hooks at 1733267677260 (+19 ms)Region opened successfully at 1733267677260 2024-12-03T23:14:37,261 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ttt,1733267676090.a6de975926600fad463e13238b2ac8a9., pid=41, masterSystemTime=1733267677085 2024-12-03T23:14:37,262 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=6 2024-12-03T23:14:37,262 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=6, state=SUCCESS, hasLock=false; OpenRegionProcedure f71f90ef2d2f59e885f2ad6ca1a99596, server=5f9d64419733,41167,1733267673270 in 228 msec 2024-12-03T23:14:37,264 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ttt,1733267676090.a6de975926600fad463e13238b2ac8a9. 2024-12-03T23:14:37,264 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(153): Opened test,ttt,1733267676090.a6de975926600fad463e13238b2ac8a9. 2024-12-03T23:14:37,265 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(132): Open test,nnn,1733267676090.f00ca5d778884039f3248393a629dcfc. 2024-12-03T23:14:37,265 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=f71f90ef2d2f59e885f2ad6ca1a99596, ASSIGN in 520 msec 2024-12-03T23:14:37,265 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7752): Opening region: {ENCODED => f00ca5d778884039f3248393a629dcfc, NAME => 'test,nnn,1733267676090.f00ca5d778884039f3248393a629dcfc.', STARTKEY => 'nnn', ENDKEY => 'ooo'} 2024-12-03T23:14:37,265 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test f00ca5d778884039f3248393a629dcfc 2024-12-03T23:14:37,265 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(898): Instantiated test,nnn,1733267676090.f00ca5d778884039f3248393a629dcfc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T23:14:37,265 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7794): checking encryption for f00ca5d778884039f3248393a629dcfc 2024-12-03T23:14:37,266 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7797): checking classloading for f00ca5d778884039f3248393a629dcfc 2024-12-03T23:14:37,266 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/default/test/89a698454f10c9fb6cfc108c35d8415a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T23:14:37,267 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=a6de975926600fad463e13238b2ac8a9, regionState=OPEN, openSeqNum=2, regionLocation=5f9d64419733,44899,1733267673202 2024-12-03T23:14:37,267 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1114): Opened 89a698454f10c9fb6cfc108c35d8415a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70319310, jitterRate=0.04783937335014343}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T23:14:37,267 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 89a698454f10c9fb6cfc108c35d8415a 2024-12-03T23:14:37,268 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1006): Region open journal for 89a698454f10c9fb6cfc108c35d8415a: Running coprocessor pre-open hook at 1733267677215Writing region info on filesystem at 1733267677215Initializing all the Stores at 1733267677218 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733267677218Cleaning up temporary data from old regions at 1733267677236 (+18 ms)Running coprocessor post-open hooks at 1733267677267 (+31 ms)Region opened successfully at 1733267677268 (+1 ms) 2024-12-03T23:14:37,269 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2236): Post open deploy tasks for test,rrr,1733267676090.89a698454f10c9fb6cfc108c35d8415a., pid=49, masterSystemTime=1733267677166 2024-12-03T23:14:37,271 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure a6de975926600fad463e13238b2ac8a9, server=5f9d64419733,44899,1733267673202 because future has completed 2024-12-03T23:14:37,271 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=742c43a49ea3b763b8232c8e8c8966b3, regionState=OPEN, openSeqNum=2, regionLocation=5f9d64419733,46833,1733267673070 2024-12-03T23:14:37,273 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2266): Finished post open deploy task for test,rrr,1733267676090.89a698454f10c9fb6cfc108c35d8415a. 2024-12-03T23:14:37,273 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(153): Opened test,rrr,1733267676090.89a698454f10c9fb6cfc108c35d8415a. 2024-12-03T23:14:37,271 INFO [StoreOpener-4e6706440af7674db97bb48a05858115-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 4e6706440af7674db97bb48a05858115 2024-12-03T23:14:37,273 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(132): Open test,eee,1733267676090.212a9963c9cddede26a4eb521cff5254. 2024-12-03T23:14:37,273 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7752): Opening region: {ENCODED => 212a9963c9cddede26a4eb521cff5254, NAME => 'test,eee,1733267676090.212a9963c9cddede26a4eb521cff5254.', STARTKEY => 'eee', ENDKEY => 'fff'} 2024-12-03T23:14:37,273 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 212a9963c9cddede26a4eb521cff5254 2024-12-03T23:14:37,273 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(898): Instantiated test,eee,1733267676090.212a9963c9cddede26a4eb521cff5254.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T23:14:37,274 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7794): checking encryption for 212a9963c9cddede26a4eb521cff5254 2024-12-03T23:14:37,274 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7797): checking classloading for 212a9963c9cddede26a4eb521cff5254 2024-12-03T23:14:37,276 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 742c43a49ea3b763b8232c8e8c8966b3, server=5f9d64419733,46833,1733267673070 because future has completed 2024-12-03T23:14:37,276 INFO [StoreOpener-4e6706440af7674db97bb48a05858115-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4e6706440af7674db97bb48a05858115 columnFamilyName family 2024-12-03T23:14:37,276 DEBUG [StoreOpener-4e6706440af7674db97bb48a05858115-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:14:37,277 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=89a698454f10c9fb6cfc108c35d8415a, regionState=OPEN, openSeqNum=2, regionLocation=5f9d64419733,41167,1733267673270 2024-12-03T23:14:37,277 INFO [StoreOpener-4e6706440af7674db97bb48a05858115-1 {}] regionserver.HStore(327): Store=4e6706440af7674db97bb48a05858115/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T23:14:37,277 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1038): replaying wal for 4e6706440af7674db97bb48a05858115 2024-12-03T23:14:37,279 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/default/test/4e6706440af7674db97bb48a05858115 2024-12-03T23:14:37,279 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/default/test/4e6706440af7674db97bb48a05858115 2024-12-03T23:14:37,280 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=49, ppid=22, state=RUNNABLE, hasLock=true; OpenRegionProcedure 89a698454f10c9fb6cfc108c35d8415a, server=5f9d64419733,41167,1733267673270 2024-12-03T23:14:37,280 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1048): stopping wal replay for 4e6706440af7674db97bb48a05858115 2024-12-03T23:14:37,280 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1060): Cleaning up temporary data for 4e6706440af7674db97bb48a05858115 2024-12-03T23:14:37,284 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1093): writing seq id for 4e6706440af7674db97bb48a05858115 2024-12-03T23:14:37,285 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=22 2024-12-03T23:14:37,285 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=22, state=SUCCESS, hasLock=false; OpenRegionProcedure 89a698454f10c9fb6cfc108c35d8415a, server=5f9d64419733,41167,1733267673270 in 262 msec 2024-12-03T23:14:37,288 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=89a698454f10c9fb6cfc108c35d8415a, ASSIGN in 543 msec 2024-12-03T23:14:37,292 INFO [StoreOpener-f00ca5d778884039f3248393a629dcfc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region f00ca5d778884039f3248393a629dcfc 2024-12-03T23:14:37,293 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=20 2024-12-03T23:14:37,293 INFO [StoreOpener-212a9963c9cddede26a4eb521cff5254-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 212a9963c9cddede26a4eb521cff5254 2024-12-03T23:14:37,293 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure 742c43a49ea3b763b8232c8e8c8966b3, server=5f9d64419733,46833,1733267673070 in 335 msec 2024-12-03T23:14:37,294 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/default/test/4e6706440af7674db97bb48a05858115/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T23:14:37,295 INFO [StoreOpener-212a9963c9cddede26a4eb521cff5254-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 212a9963c9cddede26a4eb521cff5254 columnFamilyName family 2024-12-03T23:14:37,295 DEBUG [StoreOpener-212a9963c9cddede26a4eb521cff5254-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:14:37,296 INFO [StoreOpener-212a9963c9cddede26a4eb521cff5254-1 {}] regionserver.HStore(327): Store=212a9963c9cddede26a4eb521cff5254/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T23:14:37,296 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1114): Opened 4e6706440af7674db97bb48a05858115; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64173359, jitterRate=-0.043742433190345764}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T23:14:37,296 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4e6706440af7674db97bb48a05858115 2024-12-03T23:14:37,296 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1038): replaying wal for 212a9963c9cddede26a4eb521cff5254 2024-12-03T23:14:37,297 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1006): Region open journal for 4e6706440af7674db97bb48a05858115: Running coprocessor pre-open hook at 1733267677258Writing region info on filesystem at 1733267677258Initializing all the Stores at 1733267677261 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733267677261Cleaning up temporary data from old regions at 1733267677280 (+19 ms)Running coprocessor post-open hooks at 1733267677296 (+16 ms)Region opened successfully at 1733267677296 2024-12-03T23:14:37,297 INFO [StoreOpener-f00ca5d778884039f3248393a629dcfc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f00ca5d778884039f3248393a629dcfc columnFamilyName family 2024-12-03T23:14:37,297 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=742c43a49ea3b763b8232c8e8c8966b3, ASSIGN in 551 msec 2024-12-03T23:14:37,297 DEBUG [StoreOpener-f00ca5d778884039f3248393a629dcfc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:14:37,298 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/default/test/212a9963c9cddede26a4eb521cff5254 2024-12-03T23:14:37,298 INFO [StoreOpener-f00ca5d778884039f3248393a629dcfc-1 {}] regionserver.HStore(327): Store=f00ca5d778884039f3248393a629dcfc/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T23:14:37,298 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/default/test/212a9963c9cddede26a4eb521cff5254 2024-12-03T23:14:37,299 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2236): Post open deploy tasks for test,mmm,1733267676090.4e6706440af7674db97bb48a05858115., pid=32, masterSystemTime=1733267677068 2024-12-03T23:14:37,299 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1038): replaying wal for f00ca5d778884039f3248393a629dcfc 2024-12-03T23:14:37,299 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1048): stopping wal replay for 212a9963c9cddede26a4eb521cff5254 2024-12-03T23:14:37,299 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1060): Cleaning up temporary data for 212a9963c9cddede26a4eb521cff5254 2024-12-03T23:14:37,300 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/default/test/f00ca5d778884039f3248393a629dcfc 2024-12-03T23:14:37,300 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/default/test/f00ca5d778884039f3248393a629dcfc 2024-12-03T23:14:37,301 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1048): stopping wal replay for f00ca5d778884039f3248393a629dcfc 2024-12-03T23:14:37,301 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1060): Cleaning up temporary data for f00ca5d778884039f3248393a629dcfc 2024-12-03T23:14:37,303 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2266): Finished post open deploy task for test,mmm,1733267676090.4e6706440af7674db97bb48a05858115. 2024-12-03T23:14:37,303 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(153): Opened test,mmm,1733267676090.4e6706440af7674db97bb48a05858115. 2024-12-03T23:14:37,303 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(132): Open test,,1733267676090.6d9981bd2d541debb970a37ac121e7ed. 2024-12-03T23:14:37,303 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7752): Opening region: {ENCODED => 6d9981bd2d541debb970a37ac121e7ed, NAME => 'test,,1733267676090.6d9981bd2d541debb970a37ac121e7ed.', STARTKEY => '', ENDKEY => 'bbb'} 2024-12-03T23:14:37,304 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1093): writing seq id for 212a9963c9cddede26a4eb521cff5254 2024-12-03T23:14:37,304 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 6d9981bd2d541debb970a37ac121e7ed 2024-12-03T23:14:37,304 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(898): Instantiated test,,1733267676090.6d9981bd2d541debb970a37ac121e7ed.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T23:14:37,304 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7794): checking encryption for 6d9981bd2d541debb970a37ac121e7ed 2024-12-03T23:14:37,304 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7797): checking classloading for 6d9981bd2d541debb970a37ac121e7ed 2024-12-03T23:14:37,304 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1093): writing seq id for f00ca5d778884039f3248393a629dcfc 2024-12-03T23:14:37,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-03T23:14:37,308 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/default/test/212a9963c9cddede26a4eb521cff5254/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T23:14:37,308 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=24 2024-12-03T23:14:37,308 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=24, state=SUCCESS, hasLock=false; OpenRegionProcedure a6de975926600fad463e13238b2ac8a9, server=5f9d64419733,44899,1733267673202 in 331 msec 2024-12-03T23:14:37,309 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1114): Opened 212a9963c9cddede26a4eb521cff5254; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69308596, jitterRate=0.03277856111526489}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T23:14:37,309 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 212a9963c9cddede26a4eb521cff5254 2024-12-03T23:14:37,309 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1006): Region open journal for 212a9963c9cddede26a4eb521cff5254: Running coprocessor pre-open hook at 1733267677274Writing region info on filesystem at 1733267677274Initializing all the Stores at 1733267677275 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733267677275Cleaning up temporary data from old regions at 1733267677299 (+24 ms)Running coprocessor post-open hooks at 1733267677309 (+10 ms)Region opened successfully at 1733267677309 2024-12-03T23:14:37,310 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2236): Post open deploy tasks for test,eee,1733267676090.212a9963c9cddede26a4eb521cff5254., pid=50, masterSystemTime=1733267677166 2024-12-03T23:14:37,312 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=a6de975926600fad463e13238b2ac8a9, ASSIGN in 567 msec 2024-12-03T23:14:37,313 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2266): Finished post open deploy task for test,eee,1733267676090.212a9963c9cddede26a4eb521cff5254. 2024-12-03T23:14:37,313 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(153): Opened test,eee,1733267676090.212a9963c9cddede26a4eb521cff5254. 2024-12-03T23:14:37,313 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(132): Open test,xxx,1733267676090.e96b12de28d6c3047803f54a3b027459. 2024-12-03T23:14:37,314 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7752): Opening region: {ENCODED => e96b12de28d6c3047803f54a3b027459, NAME => 'test,xxx,1733267676090.e96b12de28d6c3047803f54a3b027459.', STARTKEY => 'xxx', ENDKEY => 'yyy'} 2024-12-03T23:14:37,314 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test e96b12de28d6c3047803f54a3b027459 2024-12-03T23:14:37,314 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(898): Instantiated test,xxx,1733267676090.e96b12de28d6c3047803f54a3b027459.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T23:14:37,314 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7794): checking encryption for e96b12de28d6c3047803f54a3b027459 2024-12-03T23:14:37,314 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7797): checking classloading for e96b12de28d6c3047803f54a3b027459 2024-12-03T23:14:37,315 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=4e6706440af7674db97bb48a05858115, regionState=OPEN, openSeqNum=2, regionLocation=5f9d64419733,46833,1733267673070 2024-12-03T23:14:37,318 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4e6706440af7674db97bb48a05858115, server=5f9d64419733,46833,1733267673070 because future has completed 2024-12-03T23:14:37,318 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=212a9963c9cddede26a4eb521cff5254, regionState=OPEN, openSeqNum=2, regionLocation=5f9d64419733,41167,1733267673270 2024-12-03T23:14:37,321 INFO [StoreOpener-6d9981bd2d541debb970a37ac121e7ed-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 6d9981bd2d541debb970a37ac121e7ed 2024-12-03T23:14:37,324 INFO [StoreOpener-6d9981bd2d541debb970a37ac121e7ed-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6d9981bd2d541debb970a37ac121e7ed columnFamilyName family 2024-12-03T23:14:37,324 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=50, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure 212a9963c9cddede26a4eb521cff5254, server=5f9d64419733,41167,1733267673270 because future has completed 2024-12-03T23:14:37,324 DEBUG [StoreOpener-6d9981bd2d541debb970a37ac121e7ed-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:14:37,324 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44117 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=5f9d64419733,41167,1733267673270, table=test, region=212a9963c9cddede26a4eb521cff5254. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-03T23:14:37,326 INFO [StoreOpener-6d9981bd2d541debb970a37ac121e7ed-1 {}] regionserver.HStore(327): Store=6d9981bd2d541debb970a37ac121e7ed/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T23:14:37,327 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1038): replaying wal for 6d9981bd2d541debb970a37ac121e7ed 2024-12-03T23:14:37,329 INFO [StoreOpener-e96b12de28d6c3047803f54a3b027459-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region e96b12de28d6c3047803f54a3b027459 2024-12-03T23:14:37,329 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/default/test/f00ca5d778884039f3248393a629dcfc/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T23:14:37,330 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1114): Opened f00ca5d778884039f3248393a629dcfc; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73779785, jitterRate=0.0994044691324234}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T23:14:37,330 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f00ca5d778884039f3248393a629dcfc 2024-12-03T23:14:37,330 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1006): Region open journal for f00ca5d778884039f3248393a629dcfc: Running coprocessor pre-open hook at 1733267677266Writing region info on filesystem at 1733267677266Initializing all the Stores at 1733267677268 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733267677268Cleaning up temporary data from old regions at 1733267677301 (+33 ms)Running coprocessor post-open hooks at 1733267677330 (+29 ms)Region opened successfully at 1733267677330 2024-12-03T23:14:37,332 INFO [StoreOpener-e96b12de28d6c3047803f54a3b027459-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e96b12de28d6c3047803f54a3b027459 columnFamilyName family 2024-12-03T23:14:37,332 DEBUG [StoreOpener-e96b12de28d6c3047803f54a3b027459-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:14:37,332 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/default/test/6d9981bd2d541debb970a37ac121e7ed 2024-12-03T23:14:37,332 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2236): Post open deploy tasks for test,nnn,1733267676090.f00ca5d778884039f3248393a629dcfc., pid=42, masterSystemTime=1733267677085 2024-12-03T23:14:37,333 INFO [StoreOpener-e96b12de28d6c3047803f54a3b027459-1 {}] regionserver.HStore(327): Store=e96b12de28d6c3047803f54a3b027459/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T23:14:37,333 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=17 2024-12-03T23:14:37,333 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1038): replaying wal for e96b12de28d6c3047803f54a3b027459 2024-12-03T23:14:37,333 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=17, state=SUCCESS, hasLock=false; OpenRegionProcedure 4e6706440af7674db97bb48a05858115, server=5f9d64419733,46833,1733267673070 in 400 msec 2024-12-03T23:14:37,333 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/default/test/6d9981bd2d541debb970a37ac121e7ed 2024-12-03T23:14:37,334 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1048): stopping wal replay for 6d9981bd2d541debb970a37ac121e7ed 2024-12-03T23:14:37,334 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1060): Cleaning up temporary data for 6d9981bd2d541debb970a37ac121e7ed 2024-12-03T23:14:37,335 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/default/test/e96b12de28d6c3047803f54a3b027459 2024-12-03T23:14:37,335 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/default/test/e96b12de28d6c3047803f54a3b027459 2024-12-03T23:14:37,336 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1048): stopping wal replay for e96b12de28d6c3047803f54a3b027459 2024-12-03T23:14:37,336 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1060): Cleaning up temporary data for e96b12de28d6c3047803f54a3b027459 2024-12-03T23:14:37,337 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2266): Finished post open deploy task for test,nnn,1733267676090.f00ca5d778884039f3248393a629dcfc. 2024-12-03T23:14:37,337 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(153): Opened test,nnn,1733267676090.f00ca5d778884039f3248393a629dcfc. 2024-12-03T23:14:37,337 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(132): Open test,qqq,1733267676090.10beaf4047510d958e4a559194242a57. 2024-12-03T23:14:37,337 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7752): Opening region: {ENCODED => 10beaf4047510d958e4a559194242a57, NAME => 'test,qqq,1733267676090.10beaf4047510d958e4a559194242a57.', STARTKEY => 'qqq', ENDKEY => 'rrr'} 2024-12-03T23:14:37,337 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 10beaf4047510d958e4a559194242a57 2024-12-03T23:14:37,338 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(898): Instantiated test,qqq,1733267676090.10beaf4047510d958e4a559194242a57.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T23:14:37,338 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7794): checking encryption for 10beaf4047510d958e4a559194242a57 2024-12-03T23:14:37,338 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=4e6706440af7674db97bb48a05858115, ASSIGN in 591 msec 2024-12-03T23:14:37,338 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7797): checking classloading for 10beaf4047510d958e4a559194242a57 2024-12-03T23:14:37,339 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1093): writing seq id for 6d9981bd2d541debb970a37ac121e7ed 2024-12-03T23:14:37,340 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=f00ca5d778884039f3248393a629dcfc, regionState=OPEN, openSeqNum=2, regionLocation=5f9d64419733,44899,1733267673202 2024-12-03T23:14:37,340 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1093): writing seq id for e96b12de28d6c3047803f54a3b027459 2024-12-03T23:14:37,341 INFO [StoreOpener-10beaf4047510d958e4a559194242a57-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 10beaf4047510d958e4a559194242a57 2024-12-03T23:14:37,344 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=42, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure f00ca5d778884039f3248393a629dcfc, server=5f9d64419733,44899,1733267673202 because future has completed 2024-12-03T23:14:37,344 INFO [StoreOpener-10beaf4047510d958e4a559194242a57-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 10beaf4047510d958e4a559194242a57 columnFamilyName family 2024-12-03T23:14:37,344 DEBUG [StoreOpener-10beaf4047510d958e4a559194242a57-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:14:37,345 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/default/test/6d9981bd2d541debb970a37ac121e7ed/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T23:14:37,346 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1114): Opened 6d9981bd2d541debb970a37ac121e7ed; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75055549, jitterRate=0.11841483414173126}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T23:14:37,346 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6d9981bd2d541debb970a37ac121e7ed 2024-12-03T23:14:37,347 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1006): Region open journal for 6d9981bd2d541debb970a37ac121e7ed: Running coprocessor pre-open hook at 1733267677304Writing region info on filesystem at 1733267677304Initializing all the Stores at 1733267677309 (+5 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733267677309Cleaning up temporary data from old regions at 1733267677334 (+25 ms)Running coprocessor post-open hooks at 1733267677346 (+12 ms)Region opened successfully at 1733267677346 2024-12-03T23:14:37,347 INFO [StoreOpener-10beaf4047510d958e4a559194242a57-1 {}] regionserver.HStore(327): Store=10beaf4047510d958e4a559194242a57/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T23:14:37,348 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/default/test/e96b12de28d6c3047803f54a3b027459/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T23:14:37,348 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2236): Post open deploy tasks for test,,1733267676090.6d9981bd2d541debb970a37ac121e7ed., pid=35, masterSystemTime=1733267677068 2024-12-03T23:14:37,348 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1038): replaying wal for 10beaf4047510d958e4a559194242a57 2024-12-03T23:14:37,349 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1114): Opened e96b12de28d6c3047803f54a3b027459; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67931447, jitterRate=0.012257441878318787}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T23:14:37,349 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e96b12de28d6c3047803f54a3b027459 2024-12-03T23:14:37,349 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1006): Region open journal for e96b12de28d6c3047803f54a3b027459: Running coprocessor pre-open hook at 1733267677314Writing region info on filesystem at 1733267677314Initializing all the Stores at 1733267677316 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733267677316Cleaning up temporary data from old regions at 1733267677336 (+20 ms)Running coprocessor post-open hooks at 1733267677349 (+13 ms)Region opened successfully at 1733267677349 2024-12-03T23:14:37,350 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2236): Post open deploy tasks for test,xxx,1733267676090.e96b12de28d6c3047803f54a3b027459., pid=48, masterSystemTime=1733267677166 2024-12-03T23:14:37,350 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=50, resume processing ppid=9 2024-12-03T23:14:37,350 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, ppid=9, state=SUCCESS, hasLock=false; OpenRegionProcedure 212a9963c9cddede26a4eb521cff5254, server=5f9d64419733,41167,1733267673270 in 321 msec 2024-12-03T23:14:37,351 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/default/test/10beaf4047510d958e4a559194242a57 2024-12-03T23:14:37,351 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2266): Finished post open deploy task for test,,1733267676090.6d9981bd2d541debb970a37ac121e7ed. 2024-12-03T23:14:37,351 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(153): Opened test,,1733267676090.6d9981bd2d541debb970a37ac121e7ed. 2024-12-03T23:14:37,351 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(132): Open test,sss,1733267676090.2b7da3736c6b257b43fe66ea13807bc6. 2024-12-03T23:14:37,352 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7752): Opening region: {ENCODED => 2b7da3736c6b257b43fe66ea13807bc6, NAME => 'test,sss,1733267676090.2b7da3736c6b257b43fe66ea13807bc6.', STARTKEY => 'sss', ENDKEY => 'ttt'} 2024-12-03T23:14:37,352 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 2b7da3736c6b257b43fe66ea13807bc6 2024-12-03T23:14:37,352 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(898): Instantiated test,sss,1733267676090.2b7da3736c6b257b43fe66ea13807bc6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T23:14:37,352 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7794): checking encryption for 2b7da3736c6b257b43fe66ea13807bc6 2024-12-03T23:14:37,352 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/default/test/10beaf4047510d958e4a559194242a57 2024-12-03T23:14:37,352 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7797): checking classloading for 2b7da3736c6b257b43fe66ea13807bc6 2024-12-03T23:14:37,353 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=6d9981bd2d541debb970a37ac121e7ed, regionState=OPEN, openSeqNum=2, regionLocation=5f9d64419733,46833,1733267673070 2024-12-03T23:14:37,353 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2266): Finished post open deploy task for test,xxx,1733267676090.e96b12de28d6c3047803f54a3b027459. 2024-12-03T23:14:37,353 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(153): Opened test,xxx,1733267676090.e96b12de28d6c3047803f54a3b027459. 2024-12-03T23:14:37,353 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1048): stopping wal replay for 10beaf4047510d958e4a559194242a57 2024-12-03T23:14:37,354 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(132): Open test,ooo,1733267676090.5e6925efcdb64a8de08b4c4bbf3a3d2e. 2024-12-03T23:14:37,354 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1060): Cleaning up temporary data for 10beaf4047510d958e4a559194242a57 2024-12-03T23:14:37,354 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7752): Opening region: {ENCODED => 5e6925efcdb64a8de08b4c4bbf3a3d2e, NAME => 'test,ooo,1733267676090.5e6925efcdb64a8de08b4c4bbf3a3d2e.', STARTKEY => 'ooo', ENDKEY => 'ppp'} 2024-12-03T23:14:37,354 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 5e6925efcdb64a8de08b4c4bbf3a3d2e 2024-12-03T23:14:37,354 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(898): Instantiated test,ooo,1733267676090.5e6925efcdb64a8de08b4c4bbf3a3d2e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T23:14:37,354 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7794): checking encryption for 5e6925efcdb64a8de08b4c4bbf3a3d2e 2024-12-03T23:14:37,354 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7797): checking classloading for 5e6925efcdb64a8de08b4c4bbf3a3d2e 2024-12-03T23:14:37,357 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6d9981bd2d541debb970a37ac121e7ed, server=5f9d64419733,46833,1733267673070 because future has completed 2024-12-03T23:14:37,357 INFO [StoreOpener-5e6925efcdb64a8de08b4c4bbf3a3d2e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 5e6925efcdb64a8de08b4c4bbf3a3d2e 2024-12-03T23:14:37,358 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=e96b12de28d6c3047803f54a3b027459, regionState=OPEN, openSeqNum=2, regionLocation=5f9d64419733,41167,1733267673270 2024-12-03T23:14:37,358 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1093): writing seq id for 10beaf4047510d958e4a559194242a57 2024-12-03T23:14:37,361 INFO [StoreOpener-5e6925efcdb64a8de08b4c4bbf3a3d2e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5e6925efcdb64a8de08b4c4bbf3a3d2e columnFamilyName family 2024-12-03T23:14:37,361 DEBUG [StoreOpener-5e6925efcdb64a8de08b4c4bbf3a3d2e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:14:37,364 INFO [StoreOpener-5e6925efcdb64a8de08b4c4bbf3a3d2e-1 {}] regionserver.HStore(327): Store=5e6925efcdb64a8de08b4c4bbf3a3d2e/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T23:14:37,364 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1038): replaying wal for 5e6925efcdb64a8de08b4c4bbf3a3d2e 2024-12-03T23:14:37,365 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=48, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure e96b12de28d6c3047803f54a3b027459, server=5f9d64419733,41167,1733267673270 because future has completed 2024-12-03T23:14:37,365 INFO [StoreOpener-2b7da3736c6b257b43fe66ea13807bc6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 2b7da3736c6b257b43fe66ea13807bc6 2024-12-03T23:14:37,366 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/default/test/5e6925efcdb64a8de08b4c4bbf3a3d2e 2024-12-03T23:14:37,366 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/default/test/5e6925efcdb64a8de08b4c4bbf3a3d2e 2024-12-03T23:14:37,367 INFO [StoreOpener-2b7da3736c6b257b43fe66ea13807bc6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2b7da3736c6b257b43fe66ea13807bc6 columnFamilyName family 2024-12-03T23:14:37,367 DEBUG [StoreOpener-2b7da3736c6b257b43fe66ea13807bc6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:14:37,367 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1048): stopping wal replay for 5e6925efcdb64a8de08b4c4bbf3a3d2e 2024-12-03T23:14:37,367 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1060): Cleaning up temporary data for 5e6925efcdb64a8de08b4c4bbf3a3d2e 2024-12-03T23:14:37,368 INFO [StoreOpener-2b7da3736c6b257b43fe66ea13807bc6-1 {}] regionserver.HStore(327): Store=2b7da3736c6b257b43fe66ea13807bc6/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T23:14:37,368 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1038): replaying wal for 2b7da3736c6b257b43fe66ea13807bc6 2024-12-03T23:14:37,369 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=5 2024-12-03T23:14:37,369 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 6d9981bd2d541debb970a37ac121e7ed, server=5f9d64419733,46833,1733267673070 in 423 msec 2024-12-03T23:14:37,369 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/default/test/2b7da3736c6b257b43fe66ea13807bc6 2024-12-03T23:14:37,370 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/default/test/2b7da3736c6b257b43fe66ea13807bc6 2024-12-03T23:14:37,370 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1093): writing seq id for 5e6925efcdb64a8de08b4c4bbf3a3d2e 2024-12-03T23:14:37,370 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1048): stopping wal replay for 2b7da3736c6b257b43fe66ea13807bc6 2024-12-03T23:14:37,371 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1060): Cleaning up temporary data for 2b7da3736c6b257b43fe66ea13807bc6 2024-12-03T23:14:37,372 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=6d9981bd2d541debb970a37ac121e7ed, ASSIGN in 627 msec 2024-12-03T23:14:37,373 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1093): writing seq id for 2b7da3736c6b257b43fe66ea13807bc6 2024-12-03T23:14:37,374 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/default/test/5e6925efcdb64a8de08b4c4bbf3a3d2e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T23:14:37,375 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1114): Opened 5e6925efcdb64a8de08b4c4bbf3a3d2e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74871437, jitterRate=0.11567135155200958}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T23:14:37,375 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5e6925efcdb64a8de08b4c4bbf3a3d2e 2024-12-03T23:14:37,375 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/default/test/10beaf4047510d958e4a559194242a57/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T23:14:37,375 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1006): Region open journal for 5e6925efcdb64a8de08b4c4bbf3a3d2e: Running coprocessor pre-open hook at 1733267677355Writing region info on filesystem at 1733267677355Initializing all the Stores at 1733267677356 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733267677357 (+1 ms)Cleaning up temporary data from old regions at 1733267677367 (+10 ms)Running coprocessor post-open hooks at 1733267677375 (+8 ms)Region opened successfully at 1733267677375 2024-12-03T23:14:37,376 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1114): Opened 10beaf4047510d958e4a559194242a57; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71710929, jitterRate=0.06857611238956451}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T23:14:37,376 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 10beaf4047510d958e4a559194242a57 2024-12-03T23:14:37,376 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1006): Region open journal for 10beaf4047510d958e4a559194242a57: Running coprocessor pre-open hook at 1733267677338Writing region info on filesystem at 1733267677338Initializing all the Stores at 1733267677340 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733267677340Cleaning up temporary data from old regions at 1733267677354 (+14 ms)Running coprocessor post-open hooks at 1733267677376 (+22 ms)Region opened successfully at 1733267677376 2024-12-03T23:14:37,376 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ooo,1733267676090.5e6925efcdb64a8de08b4c4bbf3a3d2e., pid=55, masterSystemTime=1733267677166 2024-12-03T23:14:37,377 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=48, resume processing ppid=28 2024-12-03T23:14:37,378 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, ppid=28, state=SUCCESS, hasLock=false; OpenRegionProcedure e96b12de28d6c3047803f54a3b027459, server=5f9d64419733,41167,1733267673270 in 361 msec 2024-12-03T23:14:37,378 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2236): Post open deploy tasks for test,qqq,1733267676090.10beaf4047510d958e4a559194242a57., pid=47, masterSystemTime=1733267677085 2024-12-03T23:14:37,380 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ooo,1733267676090.5e6925efcdb64a8de08b4c4bbf3a3d2e. 2024-12-03T23:14:37,380 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(153): Opened test,ooo,1733267676090.5e6925efcdb64a8de08b4c4bbf3a3d2e. 2024-12-03T23:14:37,380 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(132): Open test,uuu,1733267676090.e5e4ecc042f874ec5567f7e86548256f. 2024-12-03T23:14:37,380 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7752): Opening region: {ENCODED => e5e4ecc042f874ec5567f7e86548256f, NAME => 'test,uuu,1733267676090.e5e4ecc042f874ec5567f7e86548256f.', STARTKEY => 'uuu', ENDKEY => 'vvv'} 2024-12-03T23:14:37,381 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test e5e4ecc042f874ec5567f7e86548256f 2024-12-03T23:14:37,381 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/default/test/2b7da3736c6b257b43fe66ea13807bc6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T23:14:37,381 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(898): Instantiated test,uuu,1733267676090.e5e4ecc042f874ec5567f7e86548256f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T23:14:37,381 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7794): checking encryption for e5e4ecc042f874ec5567f7e86548256f 2024-12-03T23:14:37,381 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7797): checking classloading for e5e4ecc042f874ec5567f7e86548256f 2024-12-03T23:14:37,381 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1114): Opened 2b7da3736c6b257b43fe66ea13807bc6; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74528738, jitterRate=0.11056473851203918}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T23:14:37,382 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2b7da3736c6b257b43fe66ea13807bc6 2024-12-03T23:14:37,382 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1006): Region open journal for 2b7da3736c6b257b43fe66ea13807bc6: Running coprocessor pre-open hook at 1733267677352Writing region info on filesystem at 1733267677352Initializing all the Stores at 1733267677355 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733267677355Cleaning up temporary data from old regions at 1733267677371 (+16 ms)Running coprocessor post-open hooks at 1733267677382 (+11 ms)Region opened successfully at 1733267677382 2024-12-03T23:14:37,382 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=e96b12de28d6c3047803f54a3b027459, ASSIGN in 636 msec 2024-12-03T23:14:37,382 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2266): Finished post open deploy task for test,qqq,1733267676090.10beaf4047510d958e4a559194242a57. 2024-12-03T23:14:37,382 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(153): Opened test,qqq,1733267676090.10beaf4047510d958e4a559194242a57. 2024-12-03T23:14:37,382 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open test,zzz,1733267676090.db9924063978f5acb21b073c5d94a7ad. 2024-12-03T23:14:37,383 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => db9924063978f5acb21b073c5d94a7ad, NAME => 'test,zzz,1733267676090.db9924063978f5acb21b073c5d94a7ad.', STARTKEY => 'zzz', ENDKEY => ''} 2024-12-03T23:14:37,383 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test db9924063978f5acb21b073c5d94a7ad 2024-12-03T23:14:37,383 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated test,zzz,1733267676090.db9924063978f5acb21b073c5d94a7ad.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T23:14:37,383 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for db9924063978f5acb21b073c5d94a7ad 2024-12-03T23:14:37,383 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for db9924063978f5acb21b073c5d94a7ad 2024-12-03T23:14:37,383 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=10beaf4047510d958e4a559194242a57, regionState=OPEN, openSeqNum=2, regionLocation=5f9d64419733,44899,1733267673202 2024-12-03T23:14:37,384 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2236): Post open deploy tasks for test,sss,1733267676090.2b7da3736c6b257b43fe66ea13807bc6., pid=39, masterSystemTime=1733267677068 2024-12-03T23:14:37,386 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2266): Finished post open deploy task for test,sss,1733267676090.2b7da3736c6b257b43fe66ea13807bc6. 2024-12-03T23:14:37,387 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(153): Opened test,sss,1733267676090.2b7da3736c6b257b43fe66ea13807bc6. 2024-12-03T23:14:37,387 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=47, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure 10beaf4047510d958e4a559194242a57, server=5f9d64419733,44899,1733267673202 because future has completed 2024-12-03T23:14:37,388 INFO [StoreOpener-e5e4ecc042f874ec5567f7e86548256f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region e5e4ecc042f874ec5567f7e86548256f 2024-12-03T23:14:37,389 INFO [StoreOpener-db9924063978f5acb21b073c5d94a7ad-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region db9924063978f5acb21b073c5d94a7ad 2024-12-03T23:14:37,391 INFO [StoreOpener-db9924063978f5acb21b073c5d94a7ad-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region db9924063978f5acb21b073c5d94a7ad columnFamilyName family 2024-12-03T23:14:37,391 DEBUG [StoreOpener-db9924063978f5acb21b073c5d94a7ad-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:14:37,391 INFO [StoreOpener-e5e4ecc042f874ec5567f7e86548256f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e5e4ecc042f874ec5567f7e86548256f columnFamilyName family 2024-12-03T23:14:37,391 DEBUG [StoreOpener-e5e4ecc042f874ec5567f7e86548256f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:14:37,391 INFO [StoreOpener-db9924063978f5acb21b073c5d94a7ad-1 {}] regionserver.HStore(327): Store=db9924063978f5acb21b073c5d94a7ad/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T23:14:37,392 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for db9924063978f5acb21b073c5d94a7ad 2024-12-03T23:14:37,392 INFO [StoreOpener-e5e4ecc042f874ec5567f7e86548256f-1 {}] regionserver.HStore(327): Store=e5e4ecc042f874ec5567f7e86548256f/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T23:14:37,392 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=21 2024-12-03T23:14:37,392 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1038): replaying wal for e5e4ecc042f874ec5567f7e86548256f 2024-12-03T23:14:37,392 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=21, state=SUCCESS, hasLock=false; OpenRegionProcedure 10beaf4047510d958e4a559194242a57, server=5f9d64419733,44899,1733267673202 in 387 msec 2024-12-03T23:14:37,393 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/default/test/db9924063978f5acb21b073c5d94a7ad 2024-12-03T23:14:37,393 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/default/test/e5e4ecc042f874ec5567f7e86548256f 2024-12-03T23:14:37,394 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/default/test/db9924063978f5acb21b073c5d94a7ad 2024-12-03T23:14:37,394 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/default/test/e5e4ecc042f874ec5567f7e86548256f 2024-12-03T23:14:37,394 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=10beaf4047510d958e4a559194242a57, ASSIGN in 650 msec 2024-12-03T23:14:37,395 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for db9924063978f5acb21b073c5d94a7ad 2024-12-03T23:14:37,395 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for db9924063978f5acb21b073c5d94a7ad 2024-12-03T23:14:37,395 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1048): stopping wal replay for e5e4ecc042f874ec5567f7e86548256f 2024-12-03T23:14:37,395 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1060): Cleaning up temporary data for e5e4ecc042f874ec5567f7e86548256f 2024-12-03T23:14:37,396 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=2b7da3736c6b257b43fe66ea13807bc6, regionState=OPEN, openSeqNum=2, regionLocation=5f9d64419733,46833,1733267673070 2024-12-03T23:14:37,397 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for db9924063978f5acb21b073c5d94a7ad 2024-12-03T23:14:37,398 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1093): writing seq id for e5e4ecc042f874ec5567f7e86548256f 2024-12-03T23:14:37,399 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2b7da3736c6b257b43fe66ea13807bc6, server=5f9d64419733,46833,1733267673070 because future has completed 2024-12-03T23:14:37,399 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=5e6925efcdb64a8de08b4c4bbf3a3d2e, regionState=OPEN, openSeqNum=2, regionLocation=5f9d64419733,41167,1733267673270 2024-12-03T23:14:37,401 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/default/test/db9924063978f5acb21b073c5d94a7ad/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T23:14:37,403 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/default/test/e5e4ecc042f874ec5567f7e86548256f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T23:14:37,403 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened db9924063978f5acb21b073c5d94a7ad; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74230655, jitterRate=0.1061229556798935}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T23:14:37,403 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=55, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5e6925efcdb64a8de08b4c4bbf3a3d2e, server=5f9d64419733,41167,1733267673270 because future has completed 2024-12-03T23:14:37,403 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for db9924063978f5acb21b073c5d94a7ad 2024-12-03T23:14:37,403 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for db9924063978f5acb21b073c5d94a7ad: Running coprocessor pre-open hook at 1733267677383Writing region info on filesystem at 1733267677383Initializing all the Stores at 1733267677385 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733267677385Cleaning up temporary data from old regions at 1733267677395 (+10 ms)Running coprocessor post-open hooks at 1733267677403 (+8 ms)Region opened successfully at 1733267677403 2024-12-03T23:14:37,404 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1114): Opened e5e4ecc042f874ec5567f7e86548256f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61327150, jitterRate=-0.08615425229072571}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T23:14:37,404 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e5e4ecc042f874ec5567f7e86548256f 2024-12-03T23:14:37,404 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1006): Region open journal for e5e4ecc042f874ec5567f7e86548256f: Running coprocessor pre-open hook at 1733267677381Writing region info on filesystem at 1733267677381Initializing all the Stores at 1733267677383 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733267677383Cleaning up temporary data from old regions at 1733267677395 (+12 ms)Running coprocessor post-open hooks at 1733267677404 (+9 ms)Region opened successfully at 1733267677404 2024-12-03T23:14:37,404 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for test,zzz,1733267676090.db9924063978f5acb21b073c5d94a7ad., pid=33, masterSystemTime=1733267677085 2024-12-03T23:14:37,405 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2236): Post open deploy tasks for test,uuu,1733267676090.e5e4ecc042f874ec5567f7e86548256f., pid=56, masterSystemTime=1733267677166 2024-12-03T23:14:37,407 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for test,zzz,1733267676090.db9924063978f5acb21b073c5d94a7ad. 2024-12-03T23:14:37,407 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened test,zzz,1733267676090.db9924063978f5acb21b073c5d94a7ad. 2024-12-03T23:14:37,407 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=23 2024-12-03T23:14:37,407 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=23, state=SUCCESS, hasLock=false; OpenRegionProcedure 2b7da3736c6b257b43fe66ea13807bc6, server=5f9d64419733,46833,1733267673070 in 445 msec 2024-12-03T23:14:37,407 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(132): Open test,www,1733267676090.a0f09ea7cbe3ceec5232d11c21a512a4. 2024-12-03T23:14:37,407 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7752): Opening region: {ENCODED => a0f09ea7cbe3ceec5232d11c21a512a4, NAME => 'test,www,1733267676090.a0f09ea7cbe3ceec5232d11c21a512a4.', STARTKEY => 'www', ENDKEY => 'xxx'} 2024-12-03T23:14:37,408 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test a0f09ea7cbe3ceec5232d11c21a512a4 2024-12-03T23:14:37,408 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(898): Instantiated test,www,1733267676090.a0f09ea7cbe3ceec5232d11c21a512a4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T23:14:37,408 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7794): checking encryption for a0f09ea7cbe3ceec5232d11c21a512a4 2024-12-03T23:14:37,408 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2266): Finished post open deploy task for test,uuu,1733267676090.e5e4ecc042f874ec5567f7e86548256f. 2024-12-03T23:14:37,408 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7797): checking classloading for a0f09ea7cbe3ceec5232d11c21a512a4 2024-12-03T23:14:37,408 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(153): Opened test,uuu,1733267676090.e5e4ecc042f874ec5567f7e86548256f. 2024-12-03T23:14:37,408 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(132): Open test,lll,1733267676090.1d026a7a0318e7deb79fa16b1262ae2c. 2024-12-03T23:14:37,408 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7752): Opening region: {ENCODED => 1d026a7a0318e7deb79fa16b1262ae2c, NAME => 'test,lll,1733267676090.1d026a7a0318e7deb79fa16b1262ae2c.', STARTKEY => 'lll', ENDKEY => 'mmm'} 2024-12-03T23:14:37,409 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 1d026a7a0318e7deb79fa16b1262ae2c 2024-12-03T23:14:37,409 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(898): Instantiated test,lll,1733267676090.1d026a7a0318e7deb79fa16b1262ae2c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T23:14:37,409 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7794): checking encryption for 1d026a7a0318e7deb79fa16b1262ae2c 2024-12-03T23:14:37,409 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7797): checking classloading for 1d026a7a0318e7deb79fa16b1262ae2c 2024-12-03T23:14:37,411 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=2b7da3736c6b257b43fe66ea13807bc6, ASSIGN in 665 msec 2024-12-03T23:14:37,411 INFO [StoreOpener-1d026a7a0318e7deb79fa16b1262ae2c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 1d026a7a0318e7deb79fa16b1262ae2c 2024-12-03T23:14:37,411 INFO [StoreOpener-a0f09ea7cbe3ceec5232d11c21a512a4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region a0f09ea7cbe3ceec5232d11c21a512a4 2024-12-03T23:14:37,412 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=e5e4ecc042f874ec5567f7e86548256f, regionState=OPEN, openSeqNum=2, regionLocation=5f9d64419733,41167,1733267673270 2024-12-03T23:14:37,413 INFO [StoreOpener-a0f09ea7cbe3ceec5232d11c21a512a4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a0f09ea7cbe3ceec5232d11c21a512a4 columnFamilyName family 2024-12-03T23:14:37,413 DEBUG [StoreOpener-a0f09ea7cbe3ceec5232d11c21a512a4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:14:37,414 INFO [StoreOpener-a0f09ea7cbe3ceec5232d11c21a512a4-1 {}] regionserver.HStore(327): Store=a0f09ea7cbe3ceec5232d11c21a512a4/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T23:14:37,414 INFO [StoreOpener-1d026a7a0318e7deb79fa16b1262ae2c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1d026a7a0318e7deb79fa16b1262ae2c columnFamilyName family 2024-12-03T23:14:37,414 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1038): replaying wal for a0f09ea7cbe3ceec5232d11c21a512a4 2024-12-03T23:14:37,414 DEBUG [StoreOpener-1d026a7a0318e7deb79fa16b1262ae2c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:14:37,415 INFO [StoreOpener-1d026a7a0318e7deb79fa16b1262ae2c-1 {}] regionserver.HStore(327): Store=1d026a7a0318e7deb79fa16b1262ae2c/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T23:14:37,415 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1038): replaying wal for 1d026a7a0318e7deb79fa16b1262ae2c 2024-12-03T23:14:37,416 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/default/test/a0f09ea7cbe3ceec5232d11c21a512a4 2024-12-03T23:14:37,416 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure e5e4ecc042f874ec5567f7e86548256f, server=5f9d64419733,41167,1733267673270 because future has completed 2024-12-03T23:14:37,416 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/default/test/1d026a7a0318e7deb79fa16b1262ae2c 2024-12-03T23:14:37,417 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=db9924063978f5acb21b073c5d94a7ad, regionState=OPEN, openSeqNum=2, regionLocation=5f9d64419733,44899,1733267673202 2024-12-03T23:14:37,417 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/default/test/1d026a7a0318e7deb79fa16b1262ae2c 2024-12-03T23:14:37,418 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1048): stopping wal replay for 1d026a7a0318e7deb79fa16b1262ae2c 2024-12-03T23:14:37,418 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1060): Cleaning up temporary data for 1d026a7a0318e7deb79fa16b1262ae2c 2024-12-03T23:14:37,418 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/default/test/a0f09ea7cbe3ceec5232d11c21a512a4 2024-12-03T23:14:37,420 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1048): stopping wal replay for a0f09ea7cbe3ceec5232d11c21a512a4 2024-12-03T23:14:37,420 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1060): Cleaning up temporary data for a0f09ea7cbe3ceec5232d11c21a512a4 2024-12-03T23:14:37,420 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1093): writing seq id for 1d026a7a0318e7deb79fa16b1262ae2c 2024-12-03T23:14:37,421 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure db9924063978f5acb21b073c5d94a7ad, server=5f9d64419733,44899,1733267673202 because future has completed 2024-12-03T23:14:37,422 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1093): writing seq id for a0f09ea7cbe3ceec5232d11c21a512a4 2024-12-03T23:14:37,424 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/default/test/1d026a7a0318e7deb79fa16b1262ae2c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T23:14:37,424 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1114): Opened 1d026a7a0318e7deb79fa16b1262ae2c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74779538, jitterRate=0.11430194973945618}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T23:14:37,425 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1d026a7a0318e7deb79fa16b1262ae2c 2024-12-03T23:14:37,425 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1006): Region open journal for 1d026a7a0318e7deb79fa16b1262ae2c: Running coprocessor pre-open hook at 1733267677409Writing region info on filesystem at 1733267677409Initializing all the Stores at 1733267677410 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733267677411 (+1 ms)Cleaning up temporary data from old regions at 1733267677418 (+7 ms)Running coprocessor post-open hooks at 1733267677425 (+7 ms)Region opened successfully at 1733267677425 2024-12-03T23:14:37,426 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=25 2024-12-03T23:14:37,426 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2236): Post open deploy tasks for test,lll,1733267676090.1d026a7a0318e7deb79fa16b1262ae2c., pid=54, masterSystemTime=1733267677166 2024-12-03T23:14:37,426 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=25, state=SUCCESS, hasLock=false; OpenRegionProcedure e5e4ecc042f874ec5567f7e86548256f, server=5f9d64419733,41167,1733267673270 in 373 msec 2024-12-03T23:14:37,427 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/default/test/a0f09ea7cbe3ceec5232d11c21a512a4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T23:14:37,428 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1114): Opened a0f09ea7cbe3ceec5232d11c21a512a4; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74832255, jitterRate=0.11508749425411224}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T23:14:37,428 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a0f09ea7cbe3ceec5232d11c21a512a4 2024-12-03T23:14:37,428 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44117 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=5f9d64419733,41167,1733267673270, table=test, region=5e6925efcdb64a8de08b4c4bbf3a3d2e. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-03T23:14:37,428 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1006): Region open journal for a0f09ea7cbe3ceec5232d11c21a512a4: Running coprocessor pre-open hook at 1733267677408Writing region info on filesystem at 1733267677408Initializing all the Stores at 1733267677410 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733267677411 (+1 ms)Cleaning up temporary data from old regions at 1733267677420 (+9 ms)Running coprocessor post-open hooks at 1733267677428 (+8 ms)Region opened successfully at 1733267677428 2024-12-03T23:14:37,429 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2236): Post open deploy tasks for test,www,1733267676090.a0f09ea7cbe3ceec5232d11c21a512a4., pid=34, masterSystemTime=1733267677085 2024-12-03T23:14:37,434 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=e5e4ecc042f874ec5567f7e86548256f, ASSIGN in 685 msec 2024-12-03T23:14:37,434 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2266): Finished post open deploy task for test,lll,1733267676090.1d026a7a0318e7deb79fa16b1262ae2c. 2024-12-03T23:14:37,435 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(153): Opened test,lll,1733267676090.1d026a7a0318e7deb79fa16b1262ae2c. 2024-12-03T23:14:37,435 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(132): Open test,iii,1733267676090.2db7cef12ba09a51b0a7204fc81deeec. 2024-12-03T23:14:37,435 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7752): Opening region: {ENCODED => 2db7cef12ba09a51b0a7204fc81deeec, NAME => 'test,iii,1733267676090.2db7cef12ba09a51b0a7204fc81deeec.', STARTKEY => 'iii', ENDKEY => 'jjj'} 2024-12-03T23:14:37,436 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 2db7cef12ba09a51b0a7204fc81deeec 2024-12-03T23:14:37,436 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(898): Instantiated test,iii,1733267676090.2db7cef12ba09a51b0a7204fc81deeec.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T23:14:37,436 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7794): checking encryption for 2db7cef12ba09a51b0a7204fc81deeec 2024-12-03T23:14:37,436 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7797): checking classloading for 2db7cef12ba09a51b0a7204fc81deeec 2024-12-03T23:14:37,436 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2266): Finished post open deploy task for test,www,1733267676090.a0f09ea7cbe3ceec5232d11c21a512a4. 2024-12-03T23:14:37,436 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(153): Opened test,www,1733267676090.a0f09ea7cbe3ceec5232d11c21a512a4. 2024-12-03T23:14:37,436 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44117 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=5f9d64419733,44899,1733267673202, table=test, region=f00ca5d778884039f3248393a629dcfc. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-03T23:14:37,436 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44117 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=5f9d64419733,44899,1733267673202, table=test, region=db9924063978f5acb21b073c5d94a7ad. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-03T23:14:37,437 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=1d026a7a0318e7deb79fa16b1262ae2c, regionState=OPEN, openSeqNum=2, regionLocation=5f9d64419733,41167,1733267673270 2024-12-03T23:14:37,438 INFO [StoreOpener-2db7cef12ba09a51b0a7204fc81deeec-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 2db7cef12ba09a51b0a7204fc81deeec 2024-12-03T23:14:37,440 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=54, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1d026a7a0318e7deb79fa16b1262ae2c, server=5f9d64419733,41167,1733267673270 because future has completed 2024-12-03T23:14:37,440 INFO [StoreOpener-2db7cef12ba09a51b0a7204fc81deeec-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2db7cef12ba09a51b0a7204fc81deeec columnFamilyName family 2024-12-03T23:14:37,441 DEBUG [StoreOpener-2db7cef12ba09a51b0a7204fc81deeec-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:14:37,441 INFO [StoreOpener-2db7cef12ba09a51b0a7204fc81deeec-1 {}] regionserver.HStore(327): Store=2db7cef12ba09a51b0a7204fc81deeec/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T23:14:37,441 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=a0f09ea7cbe3ceec5232d11c21a512a4, regionState=OPEN, openSeqNum=2, regionLocation=5f9d64419733,44899,1733267673202 2024-12-03T23:14:37,441 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1038): replaying wal for 2db7cef12ba09a51b0a7204fc81deeec 2024-12-03T23:14:37,442 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/default/test/2db7cef12ba09a51b0a7204fc81deeec 2024-12-03T23:14:37,444 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/default/test/2db7cef12ba09a51b0a7204fc81deeec 2024-12-03T23:14:37,444 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1048): stopping wal replay for 2db7cef12ba09a51b0a7204fc81deeec 2024-12-03T23:14:37,444 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1060): Cleaning up temporary data for 2db7cef12ba09a51b0a7204fc81deeec 2024-12-03T23:14:37,444 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure a0f09ea7cbe3ceec5232d11c21a512a4, server=5f9d64419733,44899,1733267673202 because future has completed 2024-12-03T23:14:37,447 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1093): writing seq id for 2db7cef12ba09a51b0a7204fc81deeec 2024-12-03T23:14:37,448 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=54, resume processing ppid=16 2024-12-03T23:14:37,448 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure 1d026a7a0318e7deb79fa16b1262ae2c, server=5f9d64419733,41167,1733267673270 in 403 msec 2024-12-03T23:14:37,450 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/default/test/2db7cef12ba09a51b0a7204fc81deeec/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T23:14:37,450 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=1d026a7a0318e7deb79fa16b1262ae2c, ASSIGN in 706 msec 2024-12-03T23:14:37,451 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1114): Opened 2db7cef12ba09a51b0a7204fc81deeec; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64448506, jitterRate=-0.03964242339134216}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T23:14:37,451 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2db7cef12ba09a51b0a7204fc81deeec 2024-12-03T23:14:37,451 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1006): Region open journal for 2db7cef12ba09a51b0a7204fc81deeec: Running coprocessor pre-open hook at 1733267677436Writing region info on filesystem at 1733267677436Initializing all the Stores at 1733267677438 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733267677438Cleaning up temporary data from old regions at 1733267677444 (+6 ms)Running coprocessor post-open hooks at 1733267677451 (+7 ms)Region opened successfully at 1733267677451 2024-12-03T23:14:37,453 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2236): Post open deploy tasks for test,iii,1733267676090.2db7cef12ba09a51b0a7204fc81deeec., pid=53, masterSystemTime=1733267677166 2024-12-03T23:14:37,455 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=27 2024-12-03T23:14:37,455 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=27, state=SUCCESS, hasLock=false; OpenRegionProcedure a0f09ea7cbe3ceec5232d11c21a512a4, server=5f9d64419733,44899,1733267673202 in 515 msec 2024-12-03T23:14:37,456 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2266): Finished post open deploy task for test,iii,1733267676090.2db7cef12ba09a51b0a7204fc81deeec. 2024-12-03T23:14:37,456 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(153): Opened test,iii,1733267676090.2db7cef12ba09a51b0a7204fc81deeec. 2024-12-03T23:14:37,456 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(132): Open test,ddd,1733267676090.55eff67791415c655cef877182b9bbe1. 2024-12-03T23:14:37,456 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7752): Opening region: {ENCODED => 55eff67791415c655cef877182b9bbe1, NAME => 'test,ddd,1733267676090.55eff67791415c655cef877182b9bbe1.', STARTKEY => 'ddd', ENDKEY => 'eee'} 2024-12-03T23:14:37,457 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 55eff67791415c655cef877182b9bbe1 2024-12-03T23:14:37,457 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(898): Instantiated test,ddd,1733267676090.55eff67791415c655cef877182b9bbe1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T23:14:37,457 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=a0f09ea7cbe3ceec5232d11c21a512a4, ASSIGN in 713 msec 2024-12-03T23:14:37,457 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7794): checking encryption for 55eff67791415c655cef877182b9bbe1 2024-12-03T23:14:37,457 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7797): checking classloading for 55eff67791415c655cef877182b9bbe1 2024-12-03T23:14:37,459 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=2db7cef12ba09a51b0a7204fc81deeec, regionState=OPEN, openSeqNum=2, regionLocation=5f9d64419733,41167,1733267673270 2024-12-03T23:14:37,459 INFO [StoreOpener-55eff67791415c655cef877182b9bbe1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 55eff67791415c655cef877182b9bbe1 2024-12-03T23:14:37,461 INFO [StoreOpener-55eff67791415c655cef877182b9bbe1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 55eff67791415c655cef877182b9bbe1 columnFamilyName family 2024-12-03T23:14:37,461 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=53, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2db7cef12ba09a51b0a7204fc81deeec, server=5f9d64419733,41167,1733267673270 because future has completed 2024-12-03T23:14:37,462 DEBUG [StoreOpener-55eff67791415c655cef877182b9bbe1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:14:37,462 INFO [StoreOpener-55eff67791415c655cef877182b9bbe1-1 {}] regionserver.HStore(327): Store=55eff67791415c655cef877182b9bbe1/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T23:14:37,463 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1038): replaying wal for 55eff67791415c655cef877182b9bbe1 2024-12-03T23:14:37,463 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/default/test/55eff67791415c655cef877182b9bbe1 2024-12-03T23:14:37,464 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/default/test/55eff67791415c655cef877182b9bbe1 2024-12-03T23:14:37,464 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1048): stopping wal replay for 55eff67791415c655cef877182b9bbe1 2024-12-03T23:14:37,465 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1060): Cleaning up temporary data for 55eff67791415c655cef877182b9bbe1 2024-12-03T23:14:37,465 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=30 2024-12-03T23:14:37,465 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=30, state=SUCCESS, hasLock=false; OpenRegionProcedure db9924063978f5acb21b073c5d94a7ad, server=5f9d64419733,44899,1733267673202 in 531 msec 2024-12-03T23:14:37,467 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=db9924063978f5acb21b073c5d94a7ad, ASSIGN in 723 msec 2024-12-03T23:14:37,467 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1093): writing seq id for 55eff67791415c655cef877182b9bbe1 2024-12-03T23:14:37,470 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/default/test/55eff67791415c655cef877182b9bbe1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T23:14:37,471 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1114): Opened 55eff67791415c655cef877182b9bbe1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65475316, jitterRate=-0.02434176206588745}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T23:14:37,471 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=13 2024-12-03T23:14:37,471 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 55eff67791415c655cef877182b9bbe1 2024-12-03T23:14:37,471 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=13, state=SUCCESS, hasLock=false; OpenRegionProcedure 2db7cef12ba09a51b0a7204fc81deeec, server=5f9d64419733,41167,1733267673270 in 432 msec 2024-12-03T23:14:37,471 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1006): Region open journal for 55eff67791415c655cef877182b9bbe1: Running coprocessor pre-open hook at 1733267677457Writing region info on filesystem at 1733267677457Initializing all the Stores at 1733267677459 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733267677459Cleaning up temporary data from old regions at 1733267677465 (+6 ms)Running coprocessor post-open hooks at 1733267677471 (+6 ms)Region opened successfully at 1733267677471 2024-12-03T23:14:37,472 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ddd,1733267676090.55eff67791415c655cef877182b9bbe1., pid=52, masterSystemTime=1733267677166 2024-12-03T23:14:37,474 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=2db7cef12ba09a51b0a7204fc81deeec, ASSIGN in 729 msec 2024-12-03T23:14:37,475 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ddd,1733267676090.55eff67791415c655cef877182b9bbe1. 2024-12-03T23:14:37,475 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(153): Opened test,ddd,1733267676090.55eff67791415c655cef877182b9bbe1. 2024-12-03T23:14:37,477 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=19 2024-12-03T23:14:37,477 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=19, state=SUCCESS, hasLock=false; OpenRegionProcedure 5e6925efcdb64a8de08b4c4bbf3a3d2e, server=5f9d64419733,41167,1733267673270 in 430 msec 2024-12-03T23:14:37,479 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=5e6925efcdb64a8de08b4c4bbf3a3d2e, ASSIGN in 735 msec 2024-12-03T23:14:37,480 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=55eff67791415c655cef877182b9bbe1, regionState=OPEN, openSeqNum=2, regionLocation=5f9d64419733,41167,1733267673270 2024-12-03T23:14:37,482 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=52, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 55eff67791415c655cef877182b9bbe1, server=5f9d64419733,41167,1733267673270 because future has completed 2024-12-03T23:14:37,483 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=212a9963c9cddede26a4eb521cff5254, ASSIGN in 739 msec 2024-12-03T23:14:37,486 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=52, resume processing ppid=8 2024-12-03T23:14:37,486 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 55eff67791415c655cef877182b9bbe1, server=5f9d64419733,41167,1733267673270 in 451 msec 2024-12-03T23:14:37,488 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=55eff67791415c655cef877182b9bbe1, ASSIGN in 745 msec 2024-12-03T23:14:37,491 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=42, resume processing ppid=18 2024-12-03T23:14:37,492 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, ppid=18, state=SUCCESS, hasLock=false; OpenRegionProcedure f00ca5d778884039f3248393a629dcfc, server=5f9d64419733,44899,1733267673202 in 512 msec 2024-12-03T23:14:37,495 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=4 2024-12-03T23:14:37,495 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=f00ca5d778884039f3248393a629dcfc, ASSIGN in 750 msec 2024-12-03T23:14:37,496 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-03T23:14:37,497 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733267677496"}]},"ts":"1733267677496"} 2024-12-03T23:14:37,499 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=test, state=ENABLED in hbase:meta 2024-12-03T23:14:37,500 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_POST_OPERATION 2024-12-03T23:14:37,505 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test in 1.4000 sec 2024-12-03T23:14:38,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-03T23:14:38,316 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:test completed 2024-12-03T23:14:38,316 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table test get assigned. Timeout = 60000ms 2024-12-03T23:14:38,318 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T23:14:38,341 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table test assigned to meta. Checking AM states. 2024-12-03T23:14:38,342 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T23:14:38,343 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table test assigned. 2024-12-03T23:14:38,343 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T23:14:38,351 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test,, stopping at row=test ,, for max=2147483647 with caching=100 2024-12-03T23:14:38,377 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test,, stopping at row=test ,, for max=2147483647 with caching=100 2024-12-03T23:14:38,397 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T23:14:38,399 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33960, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T23:14:38,401 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T23:14:38,403 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42244, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T23:14:38,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44117 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region f71f90ef2d2f59e885f2ad6ca1a99596 changed from -1.0 to 0.0, refreshing cache 2024-12-03T23:14:38,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44117 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 55eff67791415c655cef877182b9bbe1 changed from -1.0 to 0.0, refreshing cache 2024-12-03T23:14:38,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44117 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 212a9963c9cddede26a4eb521cff5254 changed from -1.0 to 0.0, refreshing cache 2024-12-03T23:14:38,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44117 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 2db7cef12ba09a51b0a7204fc81deeec changed from -1.0 to 0.0, refreshing cache 2024-12-03T23:14:38,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44117 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1d026a7a0318e7deb79fa16b1262ae2c changed from -1.0 to 0.0, refreshing cache 2024-12-03T23:14:38,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44117 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 5e6925efcdb64a8de08b4c4bbf3a3d2e changed from -1.0 to 0.0, refreshing cache 2024-12-03T23:14:38,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44117 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 89a698454f10c9fb6cfc108c35d8415a changed from -1.0 to 0.0, refreshing cache 2024-12-03T23:14:38,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44117 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region e5e4ecc042f874ec5567f7e86548256f changed from -1.0 to 0.0, refreshing cache 2024-12-03T23:14:38,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44117 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region e96b12de28d6c3047803f54a3b027459 changed from -1.0 to 0.0, refreshing cache 2024-12-03T23:14:38,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44117 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 0e35e57b6f6cbff01d3e782fbf6efd1f changed from -1.0 to 0.0, refreshing cache 2024-12-03T23:14:38,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44117 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region f514b27fbd5c6ef451e11852713ea906 changed from -1.0 to 0.0, refreshing cache 2024-12-03T23:14:38,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44117 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 73cd6e72f89a0d14d1d979f76c9c07a4 changed from -1.0 to 0.0, refreshing cache 2024-12-03T23:14:38,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44117 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 5ddac1aa9a6f0af338b83490d6d952c2 changed from -1.0 to 0.0, refreshing cache 2024-12-03T23:14:38,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44117 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region f00ca5d778884039f3248393a629dcfc changed from -1.0 to 0.0, refreshing cache 2024-12-03T23:14:38,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44117 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 10beaf4047510d958e4a559194242a57 changed from -1.0 to 0.0, refreshing cache 2024-12-03T23:14:38,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44117 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region a6de975926600fad463e13238b2ac8a9 changed from -1.0 to 0.0, refreshing cache 2024-12-03T23:14:38,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44117 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region a0f09ea7cbe3ceec5232d11c21a512a4 changed from -1.0 to 0.0, refreshing cache 2024-12-03T23:14:38,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44117 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region db9924063978f5acb21b073c5d94a7ad changed from -1.0 to 0.0, refreshing cache 2024-12-03T23:14:38,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44117 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 6d9981bd2d541debb970a37ac121e7ed changed from -1.0 to 0.0, refreshing cache 2024-12-03T23:14:38,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44117 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 256869bf1dc8aa33be79adb19e13d134 changed from -1.0 to 0.0, refreshing cache 2024-12-03T23:14:38,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44117 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 384b2e2791a8bf699cb7996e1aa7469c changed from -1.0 to 0.0, refreshing cache 2024-12-03T23:14:38,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44117 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 4e6706440af7674db97bb48a05858115 changed from -1.0 to 0.0, refreshing cache 2024-12-03T23:14:38,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44117 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 742c43a49ea3b763b8232c8e8c8966b3 changed from -1.0 to 0.0, refreshing cache 2024-12-03T23:14:38,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44117 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 2b7da3736c6b257b43fe66ea13807bc6 changed from -1.0 to 0.0, refreshing cache 2024-12-03T23:14:38,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44117 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 23a123891f32f583441cd7569a6de1f2 changed from -1.0 to 0.0, refreshing cache 2024-12-03T23:14:38,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44117 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 553e1b6e96399dd43d3b8f7cfad54903 changed from -1.0 to 0.0, refreshing cache 2024-12-03T23:14:38,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44117 {}] rsgroup.RSGroupBasedLoadBalancer(138): Balancing RSGroup=default 2024-12-03T23:14:38,449 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44117 {}] rsgroup.RSGroupBasedLoadBalancer(151): Start Generate Balance plan for group: default 2024-12-03T23:14:38,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44117 {}] balancer.BaseLoadBalancer(619): Start Generate Balance plan for cluster. 2024-12-03T23:14:38,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44117 {}] balancer.BalancerClusterState(204): Hosts are {5f9d64419733=0} racks are {/default-rack=0} 2024-12-03T23:14:38,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44117 {}] balancer.BalancerClusterState(303): server 0 has 9 regions 2024-12-03T23:14:38,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44117 {}] balancer.BalancerClusterState(303): server 1 has 9 regions 2024-12-03T23:14:38,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44117 {}] balancer.BalancerClusterState(303): server 2 has 9 regions 2024-12-03T23:14:38,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44117 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-03T23:14:38,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44117 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-03T23:14:38,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44117 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-12-03T23:14:38,451 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44117 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-03T23:14:38,451 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44117 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-03T23:14:38,451 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44117 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-12-03T23:14:38,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44117 {}] balancer.BalancerClusterState(326): Number of tables=2, number of hosts=1, number of racks=1 2024-12-03T23:14:38,456 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44117 {}] balancer.StochasticLoadBalancer(395): Cluster wide - skipping load balancing because weighted average imbalance=0.016474464579901153 <= threshold(0.025). If you want more aggressive balancing, either lower hbase.master.balancer.stochastic.minCostNeedBalance from 0.025 or increase the relative multiplier(s) of the specific cost function(s). functionCost=RegionCountSkewCostFunction : (multiplier=500.0, imbalance=0.0); PrimaryRegionCountSkewCostFunction : (not needed); MoveCostFunction : (multiplier=7.0, imbalance=0.0); ServerLocalityCostFunction : (multiplier=25.0, imbalance=0.0); RackLocalityCostFunction : (multiplier=15.0, imbalance=0.0); TableSkewCostFunction : (multiplier=35.0, imbalance=0.0); RegionReplicaHostCostFunction : (not needed); RegionReplicaRackCostFunction : (not needed); ReadRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); CPRequestCostFunction : (multiplier=5.0, imbalance=0.0); WriteRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); MemStoreSizeCostFunction : (multiplier=5.0, imbalance=0.0); StoreFileCostFunction : (multiplier=5.0, imbalance=0.0); 2024-12-03T23:14:38,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44117 {}] master.HMaster(2203): Balancer is going into sleep until next period in 300000ms 2024-12-03T23:14:38,459 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [900,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T23:14:38,469 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44117 {}] master.MasterRpcServices(567): Client=jenkins//172.17.0.2 set balanceSwitch=false 2024-12-03T23:14:38,490 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: master.assignment.TestRollbackSCP#testFailAndRollback Thread=378, OpenFileDescriptor=649, MaxFileDescriptor=1048576, SystemLoadAverage=558, ProcessCount=11, AvailableMemoryMB=7755 2024-12-03T23:14:38,493 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table hbase:meta 2024-12-03T23:14:38,494 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=hbase:meta,,1 2024-12-03T23:14:38,494 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(267): Killing 5f9d64419733,46833,1733267673070 2024-12-03T23:14:38,495 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server 5f9d64419733,46833,1733267673070: Simulated kill ***** 2024-12-03T23:14:38,495 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-12-03T23:14:38,497 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-12-03T23:14:38,499 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-12-03T23:14:38,500 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-12-03T23:14:38,500 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-12-03T23:14:38,500 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 315664416 }, "NonHeapMemoryUsage": { "committed": 152436736, "init": 7667712, "max": -1, "used": 150050920 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "5f9d64419733" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "5f9d64419733" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.Context": "regionserver", "tag.Hostname": "5f9d64419733" } ] } 2024-12-03T23:14:38,503 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44117 {}] master.MasterRpcServices(700): 5f9d64419733,46833,1733267673070 reported a fatal error: ***** ABORTING region server 5f9d64419733,46833,1733267673070: Simulated kill ***** 2024-12-03T23:14:38,505 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '5f9d64419733,46833,1733267673070' ***** 2024-12-03T23:14:38,505 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Simulated kill 2024-12-03T23:14:38,506 INFO [RS:0;5f9d64419733:46833 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-03T23:14:38,506 INFO [RS:0;5f9d64419733:46833 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-12-03T23:14:38,506 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-03T23:14:38,506 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [15,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T23:14:38,506 INFO [RS:0;5f9d64419733:46833 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-12-03T23:14:38,506 INFO [RS:0;5f9d64419733:46833 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-03T23:14:38,507 INFO [RS:0;5f9d64419733:46833 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;5f9d64419733:46833. 2024-12-03T23:14:38,507 DEBUG [RS:0;5f9d64419733:46833 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T23:14:38,512 DEBUG [RS:0;5f9d64419733:46833 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T23:14:38,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38883 is added to blk_1073741836_1012 (size=32114) 2024-12-03T23:14:38,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45029 is added to blk_1073741836_1012 (size=32114) 2024-12-03T23:14:38,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44333 is added to blk_1073741836_1012 (size=32114) 2024-12-03T23:14:38,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45029 is added to blk_1073741833_1009 (size=2232) 2024-12-03T23:14:38,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44333 is added to blk_1073741833_1009 (size=2232) 2024-12-03T23:14:38,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38883 is added to blk_1073741833_1009 (size=2232) 2024-12-03T23:14:38,524 DEBUG [RS:0;5f9d64419733:46833 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T23:14:38,524 INFO [RS:0;5f9d64419733:46833 {}] regionserver.LeaseManager(133): Closed leases 2024-12-03T23:14:38,524 INFO [RS:0;5f9d64419733:46833 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46833 2024-12-03T23:14:38,532 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44117-0x1017a9bb4a60000, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-03T23:14:38,532 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46833-0x1017a9bb4a60001, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/5f9d64419733,46833,1733267673070 2024-12-03T23:14:38,532 INFO [RS:0;5f9d64419733:46833 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-03T23:14:38,534 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [5f9d64419733,46833,1733267673070] 2024-12-03T23:14:38,536 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/5f9d64419733,46833,1733267673070 already deleted, retry=false 2024-12-03T23:14:38,536 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 5f9d64419733,46833,1733267673070 on 5f9d64419733,44117,1733267672173 2024-12-03T23:14:38,542 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=57, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 5f9d64419733,46833,1733267673070, splitWal=true, meta=true 2024-12-03T23:14:38,547 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=57 for 5f9d64419733,46833,1733267673070 (carryingMeta=true) 5f9d64419733,46833,1733267673070/CRASHED/regionCount=9/lock=java.util.concurrent.locks.ReentrantReadWriteLock@4ebe4cfd[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-12-03T23:14:38,548 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=57, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 5f9d64419733,46833,1733267673070, splitWal=true, meta=true 2024-12-03T23:14:38,550 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 5f9d64419733,46833,1733267673070, splitWal=true, meta=true, isMeta: true 2024-12-03T23:14:38,553 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/WALs/5f9d64419733,46833,1733267673070-splitting 2024-12-03T23:14:38,554 INFO [PEWorker-1 {}] master.SplitWALManager(105): 5f9d64419733,46833,1733267673070 WAL count=1, meta=true 2024-12-03T23:14:38,558 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 5f9d64419733%2C46833%2C1733267673070.meta.1733267675470.meta}] 2024-12-03T23:14:38,566 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=5f9d64419733,41167,1733267673270 2024-12-03T23:14:38,569 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 5f9d64419733%2C46833%2C1733267673070.meta.1733267675470.meta, worker=5f9d64419733,41167,1733267673270}] 2024-12-03T23:14:38,580 INFO [regionserver/5f9d64419733:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-03T23:14:38,608 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T23:14:38,636 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46833-0x1017a9bb4a60001, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T23:14:38,636 INFO [RS:0;5f9d64419733:46833 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-03T23:14:38,636 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46833-0x1017a9bb4a60001, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T23:14:38,636 INFO [RS:0;5f9d64419733:46833 {}] regionserver.HRegionServer(1031): Exiting; stopping=5f9d64419733,46833,1733267673070; zookeeper connection closed. 2024-12-03T23:14:38,637 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@37516cbc {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@37516cbc 2024-12-03T23:14:38,735 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=59 2024-12-03T23:14:38,759 INFO [RS_LOG_REPLAY_OPS-regionserver/5f9d64419733:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(299): Splitting hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/WALs/5f9d64419733,46833,1733267673070-splitting/5f9d64419733%2C46833%2C1733267673070.meta.1733267675470.meta, size=31.4 K (32114bytes) 2024-12-03T23:14:38,760 DEBUG [RS_LOG_REPLAY_OPS-regionserver/5f9d64419733:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-12-03T23:14:38,760 DEBUG [RS_LOG_REPLAY_OPS-regionserver/5f9d64419733:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-12-03T23:14:38,761 INFO [RS_LOG_REPLAY_OPS-regionserver/5f9d64419733:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/WALs/5f9d64419733,46833,1733267673070-splitting/5f9d64419733%2C46833%2C1733267673070.meta.1733267675470.meta 2024-12-03T23:14:38,767 INFO [RS_LOG_REPLAY_OPS-regionserver/5f9d64419733:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/WALs/5f9d64419733,46833,1733267673070-splitting/5f9d64419733%2C46833%2C1733267673070.meta.1733267675470.meta after 4ms 2024-12-03T23:14:38,777 INFO [RS_LOG_REPLAY_OPS-regionserver/5f9d64419733:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(310): Open hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/WALs/5f9d64419733,46833,1733267673070-splitting/5f9d64419733%2C46833%2C1733267673070.meta.1733267675470.meta took 19ms 2024-12-03T23:14:38,792 DEBUG [RS_LOG_REPLAY_OPS-regionserver/5f9d64419733:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(352): Last flushed sequenceid for 1588230740: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "info" sequence_id: 5 } store_sequence_id { family_name: "ns" sequence_id: 3 } store_sequence_id { family_name: "rep_barrier" sequence_id: 18446744073709551615 } store_sequence_id { family_name: "table" sequence_id: 6 } 2024-12-03T23:14:38,800 DEBUG [RS_LOG_REPLAY_OPS-regionserver/5f9d64419733:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/WALs/5f9d64419733,46833,1733267673070-splitting/5f9d64419733%2C46833%2C1733267673070.meta.1733267675470.meta so closing down 2024-12-03T23:14:38,801 DEBUG [RS_LOG_REPLAY_OPS-regionserver/5f9d64419733:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-03T23:14:38,801 INFO [RS_LOG_REPLAY_OPS-regionserver/5f9d64419733:0-0-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-5f9d64419733%2C46833%2C1733267673070.meta.1733267675470.meta.temp 2024-12-03T23:14:38,807 INFO [RS_LOG_REPLAY_OPS-regionserver/5f9d64419733:0-0-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-5f9d64419733%2C46833%2C1733267673070.meta.1733267675470.meta.temp 2024-12-03T23:14:38,811 INFO [RS_LOG_REPLAY_OPS-regionserver/5f9d64419733:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.OutputSink(145): 3 split writer threads finished 2024-12-03T23:14:38,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45029 is added to blk_1073741864_1040 (size=31873) 2024-12-03T23:14:38,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44333 is added to blk_1073741864_1040 (size=31873) 2024-12-03T23:14:38,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38883 is added to blk_1073741864_1040 (size=31873) 2024-12-03T23:14:38,823 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-5f9d64419733%2C46833%2C1733267673070.meta.1733267675470.meta.temp (wrote 57 edits, skipped 0 edits in 3 ms) 2024-12-03T23:14:38,826 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-5f9d64419733%2C46833%2C1733267673070.meta.1733267675470.meta.temp to hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-12-03T23:14:38,831 INFO [RS_LOG_REPLAY_OPS-regionserver/5f9d64419733:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(425): Processed 58 edits across 1 Regions in 46 ms; skipped=1; WAL=hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/WALs/5f9d64419733,46833,1733267673070-splitting/5f9d64419733%2C46833%2C1733267673070.meta.1733267675470.meta, size=31.4 K, length=32114, corrupted=false, cancelled=false 2024-12-03T23:14:38,831 DEBUG [RS_LOG_REPLAY_OPS-regionserver/5f9d64419733:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(428): Completed split of hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/WALs/5f9d64419733,46833,1733267673070-splitting/5f9d64419733%2C46833%2C1733267673070.meta.1733267675470.meta, journal: Splitting hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/WALs/5f9d64419733,46833,1733267673070-splitting/5f9d64419733%2C46833%2C1733267673070.meta.1733267675470.meta, size=31.4 K (32114bytes) at 1733267678759Finishing writing output for hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/WALs/5f9d64419733,46833,1733267673070-splitting/5f9d64419733%2C46833%2C1733267673070.meta.1733267675470.meta so closing down at 1733267678801 (+42 ms)Creating recovered edits writer path=hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-5f9d64419733%2C46833%2C1733267673070.meta.1733267675470.meta.temp at 1733267678807 (+6 ms)3 split writer threads finished at 1733267678811 (+4 ms)Closed recovered edits writer path=hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-5f9d64419733%2C46833%2C1733267673070.meta.1733267675470.meta.temp (wrote 57 edits, skipped 0 edits in 3 ms) at 1733267678823 (+12 ms)Rename recovered edits hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-5f9d64419733%2C46833%2C1733267673070.meta.1733267675470.meta.temp to hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 at 1733267678826 (+3 ms)Processed 58 edits across 1 Regions in 46 ms; skipped=1; WAL=hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/WALs/5f9d64419733,46833,1733267673070-splitting/5f9d64419733%2C46833%2C1733267673070.meta.1733267675470.meta, size=31.4 K, length=32114, corrupted=false, cancelled=false at 1733267678831 (+5 ms) 2024-12-03T23:14:38,832 DEBUG [RS_LOG_REPLAY_OPS-regionserver/5f9d64419733:0-0 {event_type=RS_LOG_REPLAY, pid=59}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/WALs/5f9d64419733,46833,1733267673070-splitting/5f9d64419733%2C46833%2C1733267673070.meta.1733267675470.meta 2024-12-03T23:14:38,833 DEBUG [RS_LOG_REPLAY_OPS-regionserver/5f9d64419733:0-0 {event_type=RS_LOG_REPLAY, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-12-03T23:14:38,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44117 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-12-03T23:14:38,844 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/WALs/5f9d64419733,46833,1733267673070-splitting/5f9d64419733%2C46833%2C1733267673070.meta.1733267675470.meta to hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/oldWALs 2024-12-03T23:14:38,848 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-12-03T23:14:38,848 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5f9d64419733%2C46833%2C1733267673070.meta.1733267675470.meta, worker=5f9d64419733,41167,1733267673270 in 275 msec 2024-12-03T23:14:38,850 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=5f9d64419733,41167,1733267673270 2024-12-03T23:14:38,854 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-12-03T23:14:38,854 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5f9d64419733%2C46833%2C1733267673070.meta.1733267675470.meta, worker=5f9d64419733,41167,1733267673270 in 295 msec 2024-12-03T23:14:38,857 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/WALs/5f9d64419733,46833,1733267673070-splitting dir is empty, no logs to split. 2024-12-03T23:14:38,857 INFO [PEWorker-1 {}] master.SplitWALManager(105): 5f9d64419733,46833,1733267673070 WAL count=0, meta=true 2024-12-03T23:14:38,857 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 5f9d64419733,46833,1733267673070 WAL splitting is done? wals=0, meta=true 2024-12-03T23:14:38,860 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-03T23:14:38,861 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-03T23:14:38,862 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-03T23:14:38,893 INFO [regionserver/5f9d64419733:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-03T23:14:38,893 INFO [regionserver/5f9d64419733:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-03T23:14:39,012 DEBUG [5f9d64419733:44117 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-12-03T23:14:39,013 DEBUG [5f9d64419733:44117 {}] balancer.BalancerClusterState(204): Hosts are {5f9d64419733=0} racks are {/default-rack=0} 2024-12-03T23:14:39,013 DEBUG [5f9d64419733:44117 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-03T23:14:39,013 DEBUG [5f9d64419733:44117 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-03T23:14:39,013 DEBUG [5f9d64419733:44117 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-03T23:14:39,013 DEBUG [5f9d64419733:44117 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-03T23:14:39,013 INFO [5f9d64419733:44117 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-03T23:14:39,013 INFO [5f9d64419733:44117 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-03T23:14:39,013 DEBUG [5f9d64419733:44117 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-03T23:14:39,013 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=5f9d64419733,41167,1733267673270 2024-12-03T23:14:39,015 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 5f9d64419733,41167,1733267673270, state=OPENING 2024-12-03T23:14:39,017 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44117-0x1017a9bb4a60000, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T23:14:39,017 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44899-0x1017a9bb4a60002, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T23:14:39,017 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41167-0x1017a9bb4a60003, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T23:14:39,017 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-03T23:14:39,017 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T23:14:39,017 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T23:14:39,017 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=5f9d64419733,41167,1733267673270}] 2024-12-03T23:14:39,018 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T23:14:39,176 INFO [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=61}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-03T23:14:39,176 INFO [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T23:14:39,177 INFO [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=61}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-03T23:14:39,179 INFO [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=5f9d64419733%2C41167%2C1733267673270.meta, suffix=.meta, logDir=hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/WALs/5f9d64419733,41167,1733267673270, archiveDir=hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/oldWALs, maxLogs=32 2024-12-03T23:14:39,194 DEBUG [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=61}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/WALs/5f9d64419733,41167,1733267673270/5f9d64419733%2C41167%2C1733267673270.meta.1733267679180.meta, exclude list is [], retry=0 2024-12-03T23:14:39,197 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44333,DS-e710ddc2-08f6-4b71-b4db-e94cb5850278,DISK] 2024-12-03T23:14:39,197 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38883,DS-91eef8b1-4e86-469d-90a0-bef7307996a0,DISK] 2024-12-03T23:14:39,197 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45029,DS-e96a58d6-577c-4a7d-bf1d-88064ab38370,DISK] 2024-12-03T23:14:39,200 INFO [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/WALs/5f9d64419733,41167,1733267673270/5f9d64419733%2C41167%2C1733267673270.meta.1733267679180.meta 2024-12-03T23:14:39,200 DEBUG [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39967:39967),(127.0.0.1/127.0.0.1:37111:37111),(127.0.0.1/127.0.0.1:34163:34163)] 2024-12-03T23:14:39,200 DEBUG [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-03T23:14:39,201 DEBUG [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=61}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-03T23:14:39,201 DEBUG [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-03T23:14:39,201 INFO [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-03T23:14:39,201 DEBUG [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-03T23:14:39,201 DEBUG [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T23:14:39,201 DEBUG [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-03T23:14:39,201 DEBUG [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-03T23:14:39,203 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-03T23:14:39,204 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-03T23:14:39,204 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:14:39,205 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T23:14:39,205 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-03T23:14:39,206 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-03T23:14:39,206 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:14:39,206 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T23:14:39,206 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-03T23:14:39,207 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-03T23:14:39,207 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:14:39,208 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T23:14:39,208 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-03T23:14:39,209 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-03T23:14:39,209 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:14:39,209 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T23:14:39,209 DEBUG [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-03T23:14:39,210 DEBUG [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/hbase/meta/1588230740 2024-12-03T23:14:39,212 DEBUG [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/hbase/meta/1588230740 2024-12-03T23:14:39,213 INFO [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-12-03T23:14:39,227 DEBUG [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5793): Applied 264, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=60, path=hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-12-03T23:14:39,230 INFO [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=28 KB heapSize=48.31 KB 2024-12-03T23:14:39,293 DEBUG [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/hbase/meta/1588230740/.tmp/info/1f1911448db24d25a8f334deddce552f is 138, key is test,bbb,1733267676090.f71f90ef2d2f59e885f2ad6ca1a99596./info:regioninfo/1733267677234/Put/seqid=0 2024-12-03T23:14:39,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38883 is added to blk_1073741866_1042 (size=34924) 2024-12-03T23:14:39,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44333 is added to blk_1073741866_1042 (size=34924) 2024-12-03T23:14:39,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45029 is added to blk_1073741866_1042 (size=34924) 2024-12-03T23:14:39,317 INFO [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=27.86 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/hbase/meta/1588230740/.tmp/info/1f1911448db24d25a8f334deddce552f 2024-12-03T23:14:39,385 DEBUG [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/hbase/meta/1588230740/.tmp/ns/a2e93dd1d90a4ff2a4f8a3701eae8224 is 43, key is default/ns:d/1733267675737/Put/seqid=0 2024-12-03T23:14:39,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44333 is added to blk_1073741867_1043 (size=5153) 2024-12-03T23:14:39,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45029 is added to blk_1073741867_1043 (size=5153) 2024-12-03T23:14:39,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38883 is added to blk_1073741867_1043 (size=5153) 2024-12-03T23:14:39,397 INFO [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/hbase/meta/1588230740/.tmp/ns/a2e93dd1d90a4ff2a4f8a3701eae8224 2024-12-03T23:14:39,423 DEBUG [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/hbase/meta/1588230740/.tmp/table/03c02745a36a4e5a95b73ee1e265d054 is 40, key is test/table:state/1733267677496/Put/seqid=0 2024-12-03T23:14:39,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38883 is added to blk_1073741868_1044 (size=5165) 2024-12-03T23:14:39,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44333 is added to blk_1073741868_1044 (size=5165) 2024-12-03T23:14:39,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45029 is added to blk_1073741868_1044 (size=5165) 2024-12-03T23:14:39,433 INFO [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=72 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/hbase/meta/1588230740/.tmp/table/03c02745a36a4e5a95b73ee1e265d054 2024-12-03T23:14:39,442 DEBUG [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/hbase/meta/1588230740/.tmp/info/1f1911448db24d25a8f334deddce552f as hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/hbase/meta/1588230740/info/1f1911448db24d25a8f334deddce552f 2024-12-03T23:14:39,453 INFO [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/hbase/meta/1588230740/info/1f1911448db24d25a8f334deddce552f, entries=260, sequenceid=60, filesize=34.1 K 2024-12-03T23:14:39,457 DEBUG [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/hbase/meta/1588230740/.tmp/ns/a2e93dd1d90a4ff2a4f8a3701eae8224 as hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/hbase/meta/1588230740/ns/a2e93dd1d90a4ff2a4f8a3701eae8224 2024-12-03T23:14:39,467 INFO [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/hbase/meta/1588230740/ns/a2e93dd1d90a4ff2a4f8a3701eae8224, entries=2, sequenceid=60, filesize=5.0 K 2024-12-03T23:14:39,468 DEBUG [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/hbase/meta/1588230740/.tmp/table/03c02745a36a4e5a95b73ee1e265d054 as hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/hbase/meta/1588230740/table/03c02745a36a4e5a95b73ee1e265d054 2024-12-03T23:14:39,476 INFO [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/hbase/meta/1588230740/table/03c02745a36a4e5a95b73ee1e265d054, entries=2, sequenceid=60, filesize=5.0 K 2024-12-03T23:14:39,479 INFO [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~28 KB/28672, heapSize ~48.02 KB/49168, currentSize=0 B/0 for 1588230740 in 248ms, sequenceid=60, compaction requested=false; wal=null 2024-12-03T23:14:39,480 DEBUG [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-03T23:14:39,483 DEBUG [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-12-03T23:14:39,485 DEBUG [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-03T23:14:39,485 DEBUG [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-03T23:14:39,485 DEBUG [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-03T23:14:39,487 DEBUG [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-03T23:14:39,490 DEBUG [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/hbase/meta/1588230740/recovered.edits/60.seqid, newMaxSeqId=60, maxSeqId=1 2024-12-03T23:14:39,491 INFO [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=61; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70138464, jitterRate=0.04514455795288086}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-03T23:14:39,492 DEBUG [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-03T23:14:39,492 DEBUG [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733267679202Writing region info on filesystem at 1733267679202Initializing all the Stores at 1733267679203 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733267679203Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733267679203Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733267679203Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733267679203Obtaining lock to block concurrent updates at 1733267679230 (+27 ms)Preparing flush snapshotting stores in 1588230740 at 1733267679230Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=28672, getHeapSize=49408, getOffHeapSize=0, getCellsCount=264 at 1733267679233 (+3 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733267679233Flushing 1588230740/info: creating writer at 1733267679235 (+2 ms)Flushing 1588230740/info: appending metadata at 1733267679284 (+49 ms)Flushing 1588230740/info: closing flushed file at 1733267679287 (+3 ms)Flushing 1588230740/ns: creating writer at 1733267679366 (+79 ms)Flushing 1588230740/ns: appending metadata at 1733267679383 (+17 ms)Flushing 1588230740/ns: closing flushed file at 1733267679383Flushing 1588230740/table: creating writer at 1733267679405 (+22 ms)Flushing 1588230740/table: appending metadata at 1733267679421 (+16 ms)Flushing 1588230740/table: closing flushed file at 1733267679421Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@73d0940c: reopening flushed file at 1733267679441 (+20 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@799da2e9: reopening flushed file at 1733267679456 (+15 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@620ad9cc: reopening flushed file at 1733267679467 (+11 ms)Finished flush of dataSize ~28 KB/28672, heapSize ~48.02 KB/49168, currentSize=0 B/0 for 1588230740 in 248ms, sequenceid=60, compaction requested=false; wal=null at 1733267679479 (+12 ms)Cleaning up temporary data from old regions at 1733267679485 (+6 ms)Running coprocessor post-open hooks at 1733267679492 (+7 ms)Region opened successfully at 1733267679492 2024-12-03T23:14:39,493 INFO [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=61, masterSystemTime=1733267679170 2024-12-03T23:14:39,496 DEBUG [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-03T23:14:39,496 INFO [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=61}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-03T23:14:39,497 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=61, regionLocation=5f9d64419733,41167,1733267673270 2024-12-03T23:14:39,498 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 5f9d64419733,41167,1733267673270, state=OPEN 2024-12-03T23:14:39,500 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41167-0x1017a9bb4a60003, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T23:14:39,500 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44117-0x1017a9bb4a60000, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T23:14:39,500 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44899-0x1017a9bb4a60002, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T23:14:39,500 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T23:14:39,500 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T23:14:39,500 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T23:14:39,501 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=61, ppid=60, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=5f9d64419733,41167,1733267673270 2024-12-03T23:14:39,504 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-12-03T23:14:39,505 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=5f9d64419733,41167,1733267673270 in 484 msec 2024-12-03T23:14:39,507 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=60, resume processing ppid=57 2024-12-03T23:14:39,507 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 646 msec 2024-12-03T23:14:39,508 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): 5f9d64419733,46833,1733267673070 had 9 regions 2024-12-03T23:14:39,509 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 5f9d64419733,46833,1733267673070, splitWal=true, meta=true, isMeta: false 2024-12-03T23:14:39,511 INFO [PEWorker-1 {}] master.SplitWALManager(105): 5f9d64419733,46833,1733267673070 WAL count=1, meta=false 2024-12-03T23:14:39,511 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=57, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 5f9d64419733%2C46833%2C1733267673070.1733267675041}] 2024-12-03T23:14:39,513 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=5f9d64419733,41167,1733267673270 2024-12-03T23:14:39,515 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 5f9d64419733%2C46833%2C1733267673070.1733267675041, worker=5f9d64419733,41167,1733267673270}] 2024-12-03T23:14:39,669 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=63 2024-12-03T23:14:39,686 INFO [RS_LOG_REPLAY_OPS-regionserver/5f9d64419733:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(299): Splitting hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/WALs/5f9d64419733,46833,1733267673070-splitting/5f9d64419733%2C46833%2C1733267673070.1733267675041, size=2.2 K (2232bytes) 2024-12-03T23:14:39,686 INFO [RS_LOG_REPLAY_OPS-regionserver/5f9d64419733:0-1 {event_type=RS_LOG_REPLAY, pid=63}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/WALs/5f9d64419733,46833,1733267673070-splitting/5f9d64419733%2C46833%2C1733267673070.1733267675041 2024-12-03T23:14:39,686 INFO [RS_LOG_REPLAY_OPS-regionserver/5f9d64419733:0-1 {event_type=RS_LOG_REPLAY, pid=63}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/WALs/5f9d64419733,46833,1733267673070-splitting/5f9d64419733%2C46833%2C1733267673070.1733267675041 after 0ms 2024-12-03T23:14:39,690 INFO [RS_LOG_REPLAY_OPS-regionserver/5f9d64419733:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(310): Open hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/WALs/5f9d64419733,46833,1733267673070-splitting/5f9d64419733%2C46833%2C1733267673070.1733267675041 took 5ms 2024-12-03T23:14:39,693 DEBUG [RS_LOG_REPLAY_OPS-regionserver/5f9d64419733:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 384b2e2791a8bf699cb7996e1aa7469c: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-12-03T23:14:39,695 DEBUG [RS_LOG_REPLAY_OPS-regionserver/5f9d64419733:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 553e1b6e96399dd43d3b8f7cfad54903: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-12-03T23:14:39,696 DEBUG [RS_LOG_REPLAY_OPS-regionserver/5f9d64419733:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 256869bf1dc8aa33be79adb19e13d134: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-12-03T23:14:39,697 DEBUG [RS_LOG_REPLAY_OPS-regionserver/5f9d64419733:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 23a123891f32f583441cd7569a6de1f2: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-12-03T23:14:39,699 DEBUG [RS_LOG_REPLAY_OPS-regionserver/5f9d64419733:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 742c43a49ea3b763b8232c8e8c8966b3: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-12-03T23:14:39,700 DEBUG [RS_LOG_REPLAY_OPS-regionserver/5f9d64419733:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 4e6706440af7674db97bb48a05858115: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-12-03T23:14:39,701 DEBUG [RS_LOG_REPLAY_OPS-regionserver/5f9d64419733:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 6d9981bd2d541debb970a37ac121e7ed: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-12-03T23:14:39,703 DEBUG [RS_LOG_REPLAY_OPS-regionserver/5f9d64419733:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 2b7da3736c6b257b43fe66ea13807bc6: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-12-03T23:14:39,703 DEBUG [RS_LOG_REPLAY_OPS-regionserver/5f9d64419733:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/WALs/5f9d64419733,46833,1733267673070-splitting/5f9d64419733%2C46833%2C1733267673070.1733267675041 so closing down 2024-12-03T23:14:39,703 DEBUG [RS_LOG_REPLAY_OPS-regionserver/5f9d64419733:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-03T23:14:39,703 INFO [RS_LOG_REPLAY_OPS-regionserver/5f9d64419733:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.OutputSink(145): 3 split writer threads finished 2024-12-03T23:14:39,703 INFO [RS_LOG_REPLAY_OPS-regionserver/5f9d64419733:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(425): Processed 8 edits across 0 Regions in 13 ms; skipped=8; WAL=hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/WALs/5f9d64419733,46833,1733267673070-splitting/5f9d64419733%2C46833%2C1733267673070.1733267675041, size=2.2 K, length=2232, corrupted=false, cancelled=false 2024-12-03T23:14:39,703 DEBUG [RS_LOG_REPLAY_OPS-regionserver/5f9d64419733:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(428): Completed split of hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/WALs/5f9d64419733,46833,1733267673070-splitting/5f9d64419733%2C46833%2C1733267673070.1733267675041, journal: Splitting hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/WALs/5f9d64419733,46833,1733267673070-splitting/5f9d64419733%2C46833%2C1733267673070.1733267675041, size=2.2 K (2232bytes) at 1733267679686Finishing writing output for hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/WALs/5f9d64419733,46833,1733267673070-splitting/5f9d64419733%2C46833%2C1733267673070.1733267675041 so closing down at 1733267679703 (+17 ms)3 split writer threads finished at 1733267679703Processed 8 edits across 0 Regions in 13 ms; skipped=8; WAL=hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/WALs/5f9d64419733,46833,1733267673070-splitting/5f9d64419733%2C46833%2C1733267673070.1733267675041, size=2.2 K, length=2232, corrupted=false, cancelled=false at 1733267679703 2024-12-03T23:14:39,703 DEBUG [RS_LOG_REPLAY_OPS-regionserver/5f9d64419733:0-1 {event_type=RS_LOG_REPLAY, pid=63}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/WALs/5f9d64419733,46833,1733267673070-splitting/5f9d64419733%2C46833%2C1733267673070.1733267675041 2024-12-03T23:14:39,703 DEBUG [RS_LOG_REPLAY_OPS-regionserver/5f9d64419733:0-1 {event_type=RS_LOG_REPLAY, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-12-03T23:14:39,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44117 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-12-03T23:14:39,709 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/WALs/5f9d64419733,46833,1733267673070-splitting/5f9d64419733%2C46833%2C1733267673070.1733267675041 to hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/oldWALs 2024-12-03T23:14:39,712 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-12-03T23:14:39,712 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5f9d64419733%2C46833%2C1733267673070.1733267675041, worker=5f9d64419733,41167,1733267673270 in 194 msec 2024-12-03T23:14:39,714 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=5f9d64419733,41167,1733267673270 2024-12-03T23:14:39,717 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=57 2024-12-03T23:14:39,717 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5f9d64419733%2C46833%2C1733267673070.1733267675041, worker=5f9d64419733,41167,1733267673270 in 203 msec 2024-12-03T23:14:39,720 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/WALs/5f9d64419733,46833,1733267673070-splitting dir is empty, no logs to split. 2024-12-03T23:14:39,720 INFO [PEWorker-1 {}] master.SplitWALManager(105): 5f9d64419733,46833,1733267673070 WAL count=0, meta=false 2024-12-03T23:14:39,720 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 5f9d64419733,46833,1733267673070 WAL splitting is done? wals=0, meta=false 2024-12-03T23:14:39,722 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=23a123891f32f583441cd7569a6de1f2, ASSIGN}, {pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2b7da3736c6b257b43fe66ea13807bc6, ASSIGN}, {pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4e6706440af7674db97bb48a05858115, ASSIGN}, {pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=256869bf1dc8aa33be79adb19e13d134, ASSIGN}, {pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=742c43a49ea3b763b8232c8e8c8966b3, ASSIGN}, {pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=553e1b6e96399dd43d3b8f7cfad54903, ASSIGN}, {pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=384b2e2791a8bf699cb7996e1aa7469c, ASSIGN}, {pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=6d9981bd2d541debb970a37ac121e7ed, ASSIGN}] 2024-12-03T23:14:39,724 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=6d9981bd2d541debb970a37ac121e7ed, ASSIGN 2024-12-03T23:14:39,725 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=6d9981bd2d541debb970a37ac121e7ed, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-03T23:14:39,726 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=384b2e2791a8bf699cb7996e1aa7469c, ASSIGN 2024-12-03T23:14:39,726 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=384b2e2791a8bf699cb7996e1aa7469c, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-03T23:14:39,727 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=553e1b6e96399dd43d3b8f7cfad54903, ASSIGN 2024-12-03T23:14:39,728 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=553e1b6e96399dd43d3b8f7cfad54903, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-03T23:14:39,729 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=742c43a49ea3b763b8232c8e8c8966b3, ASSIGN 2024-12-03T23:14:39,730 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=742c43a49ea3b763b8232c8e8c8966b3, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-03T23:14:39,731 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=256869bf1dc8aa33be79adb19e13d134, ASSIGN 2024-12-03T23:14:39,731 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=256869bf1dc8aa33be79adb19e13d134, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-03T23:14:39,732 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4e6706440af7674db97bb48a05858115, ASSIGN 2024-12-03T23:14:39,733 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=4e6706440af7674db97bb48a05858115, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-03T23:14:39,734 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2b7da3736c6b257b43fe66ea13807bc6, ASSIGN 2024-12-03T23:14:39,735 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=2b7da3736c6b257b43fe66ea13807bc6, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-03T23:14:39,736 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=23a123891f32f583441cd7569a6de1f2, ASSIGN 2024-12-03T23:14:39,736 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=23a123891f32f583441cd7569a6de1f2, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-03T23:14:39,875 DEBUG [5f9d64419733:44117 {}] balancer.BalancerClusterState(204): Hosts are {5f9d64419733=0} racks are {/default-rack=0} 2024-12-03T23:14:39,876 DEBUG [5f9d64419733:44117 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-03T23:14:39,876 DEBUG [5f9d64419733:44117 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-03T23:14:39,876 DEBUG [5f9d64419733:44117 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-03T23:14:39,876 DEBUG [5f9d64419733:44117 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-03T23:14:39,876 INFO [5f9d64419733:44117 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-03T23:14:39,876 INFO [5f9d64419733:44117 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-03T23:14:39,876 DEBUG [5f9d64419733:44117 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-03T23:14:39,877 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=23a123891f32f583441cd7569a6de1f2, regionState=OPENING, regionLocation=5f9d64419733,44899,1733267673202 2024-12-03T23:14:39,877 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=68 updating hbase:meta row=742c43a49ea3b763b8232c8e8c8966b3, regionState=OPENING, regionLocation=5f9d64419733,44899,1733267673202 2024-12-03T23:14:39,878 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=70 updating hbase:meta row=384b2e2791a8bf699cb7996e1aa7469c, regionState=OPENING, regionLocation=5f9d64419733,44899,1733267673202 2024-12-03T23:14:39,878 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 5f9d64419733:46833 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 5f9d64419733/172.17.0.2:46833 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T23:14:39,879 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=71 updating hbase:meta row=6d9981bd2d541debb970a37ac121e7ed, regionState=OPENING, regionLocation=5f9d64419733,44899,1733267673202 2024-12-03T23:14:39,879 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=69 updating hbase:meta row=553e1b6e96399dd43d3b8f7cfad54903, regionState=OPENING, regionLocation=5f9d64419733,41167,1733267673270 2024-12-03T23:14:39,880 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=5f9d64419733,46833,1733267673070, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=5f9d64419733,46833,1733267673070, seqNum=-1, error=java.net.ConnectException: Call to address=5f9d64419733:46833 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 5f9d64419733/172.17.0.2:46833 2024-12-03T23:14:39,880 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=65 updating hbase:meta row=2b7da3736c6b257b43fe66ea13807bc6, regionState=OPENING, regionLocation=5f9d64419733,41167,1733267673270 2024-12-03T23:14:39,880 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=5f9d64419733,46833,1733267673070, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-12-03T23:14:39,880 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=5f9d64419733,46833,1733267673070, seqNum=-1 from cache 2024-12-03T23:14:39,881 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=66 updating hbase:meta row=4e6706440af7674db97bb48a05858115, regionState=OPENING, regionLocation=5f9d64419733,41167,1733267673270 2024-12-03T23:14:39,881 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T23:14:39,881 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5f9d64419733,41167,1733267673270, seqNum=-1] 2024-12-03T23:14:39,881 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 5f9d64419733:46833 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 5f9d64419733/172.17.0.2:46833 2024-12-03T23:14:39,881 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T23:14:39,882 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 5f9d64419733:46833 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 5f9d64419733/172.17.0.2:46833 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T23:14:39,882 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=67 updating hbase:meta row=256869bf1dc8aa33be79adb19e13d134, regionState=OPENING, regionLocation=5f9d64419733,41167,1733267673270 2024-12-03T23:14:39,882 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=5f9d64419733,46833,1733267673070, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=5f9d64419733,41167,1733267673270, seqNum=-1, error=java.net.ConnectException: Call to address=5f9d64419733:46833 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 5f9d64419733/172.17.0.2:46833 2024-12-03T23:14:39,883 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=5f9d64419733,46833,1733267673070, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=5f9d64419733,41167,1733267673270, seqNum=-1, error=java.net.ConnectException: Call to address=5f9d64419733:46833 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 5f9d64419733/172.17.0.2:46833 2024-12-03T23:14:39,883 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=5f9d64419733,46833,1733267673070, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=5f9d64419733,41167,1733267673270, seqNum=-1, error=java.net.ConnectException: Call to address=5f9d64419733:46833 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 5f9d64419733/172.17.0.2:46833 2024-12-03T23:14:39,883 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=5f9d64419733,46833,1733267673070, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=5f9d64419733,41167,1733267673270, seqNum=-1, error=java.net.ConnectException: Call to address=5f9d64419733:46833 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 5f9d64419733/172.17.0.2:46833 2024-12-03T23:14:39,883 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=5f9d64419733,46833,1733267673070, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=5f9d64419733,41167,1733267673270, seqNum=-1, error=java.net.ConnectException: Call to address=5f9d64419733:46833 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 5f9d64419733/172.17.0.2:46833 2024-12-03T23:14:39,883 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 5f9d64419733:46833 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 5f9d64419733/172.17.0.2:46833 2024-12-03T23:14:39,884 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37917, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T23:14:39,886 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=4e6706440af7674db97bb48a05858115, ASSIGN because future has completed 2024-12-03T23:14:39,887 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=72, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4e6706440af7674db97bb48a05858115, server=5f9d64419733,41167,1733267673270}] 2024-12-03T23:14:39,888 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=256869bf1dc8aa33be79adb19e13d134, ASSIGN because future has completed 2024-12-03T23:14:39,891 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 256869bf1dc8aa33be79adb19e13d134, server=5f9d64419733,41167,1733267673270}] 2024-12-03T23:14:39,987 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=23a123891f32f583441cd7569a6de1f2, ASSIGN because future has completed 2024-12-03T23:14:39,988 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=74, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 23a123891f32f583441cd7569a6de1f2, server=5f9d64419733,44899,1733267673202}] 2024-12-03T23:14:39,988 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=742c43a49ea3b763b8232c8e8c8966b3, ASSIGN because future has completed 2024-12-03T23:14:39,990 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=6d9981bd2d541debb970a37ac121e7ed, ASSIGN because future has completed 2024-12-03T23:14:39,991 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=2b7da3736c6b257b43fe66ea13807bc6, ASSIGN because future has completed 2024-12-03T23:14:39,992 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2b7da3736c6b257b43fe66ea13807bc6, server=5f9d64419733,41167,1733267673270}] 2024-12-03T23:14:39,996 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=76, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6d9981bd2d541debb970a37ac121e7ed, server=5f9d64419733,44899,1733267673202}] 2024-12-03T23:14:39,997 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=384b2e2791a8bf699cb7996e1aa7469c, ASSIGN because future has completed 2024-12-03T23:14:39,998 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=553e1b6e96399dd43d3b8f7cfad54903, ASSIGN because future has completed 2024-12-03T23:14:40,000 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 553e1b6e96399dd43d3b8f7cfad54903, server=5f9d64419733,41167,1733267673270}] 2024-12-03T23:14:40,008 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=78, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 384b2e2791a8bf699cb7996e1aa7469c, server=5f9d64419733,44899,1733267673202}] 2024-12-03T23:14:40,012 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 742c43a49ea3b763b8232c8e8c8966b3, server=5f9d64419733,44899,1733267673202}] 2024-12-03T23:14:40,045 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] handler.AssignRegionHandler(132): Open test,hhh,1733267676090.256869bf1dc8aa33be79adb19e13d134. 2024-12-03T23:14:40,045 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(7752): Opening region: {ENCODED => 256869bf1dc8aa33be79adb19e13d134, NAME => 'test,hhh,1733267676090.256869bf1dc8aa33be79adb19e13d134.', STARTKEY => 'hhh', ENDKEY => 'iii'} 2024-12-03T23:14:40,045 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 256869bf1dc8aa33be79adb19e13d134 2024-12-03T23:14:40,045 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(898): Instantiated test,hhh,1733267676090.256869bf1dc8aa33be79adb19e13d134.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T23:14:40,045 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(7794): checking encryption for 256869bf1dc8aa33be79adb19e13d134 2024-12-03T23:14:40,045 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(7797): checking classloading for 256869bf1dc8aa33be79adb19e13d134 2024-12-03T23:14:40,047 INFO [StoreOpener-256869bf1dc8aa33be79adb19e13d134-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 256869bf1dc8aa33be79adb19e13d134 2024-12-03T23:14:40,049 INFO [StoreOpener-256869bf1dc8aa33be79adb19e13d134-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 256869bf1dc8aa33be79adb19e13d134 columnFamilyName family 2024-12-03T23:14:40,049 DEBUG [StoreOpener-256869bf1dc8aa33be79adb19e13d134-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:14:40,049 INFO [StoreOpener-256869bf1dc8aa33be79adb19e13d134-1 {}] regionserver.HStore(327): Store=256869bf1dc8aa33be79adb19e13d134/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T23:14:40,050 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1038): replaying wal for 256869bf1dc8aa33be79adb19e13d134 2024-12-03T23:14:40,051 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/default/test/256869bf1dc8aa33be79adb19e13d134 2024-12-03T23:14:40,052 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/default/test/256869bf1dc8aa33be79adb19e13d134 2024-12-03T23:14:40,053 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1048): stopping wal replay for 256869bf1dc8aa33be79adb19e13d134 2024-12-03T23:14:40,053 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1060): Cleaning up temporary data for 256869bf1dc8aa33be79adb19e13d134 2024-12-03T23:14:40,055 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1093): writing seq id for 256869bf1dc8aa33be79adb19e13d134 2024-12-03T23:14:40,057 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1114): Opened 256869bf1dc8aa33be79adb19e13d134; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72873072, jitterRate=0.08589339256286621}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T23:14:40,057 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 256869bf1dc8aa33be79adb19e13d134 2024-12-03T23:14:40,057 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1006): Region open journal for 256869bf1dc8aa33be79adb19e13d134: Running coprocessor pre-open hook at 1733267680046Writing region info on filesystem at 1733267680046Initializing all the Stores at 1733267680047 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733267680047Cleaning up temporary data from old regions at 1733267680053 (+6 ms)Running coprocessor post-open hooks at 1733267680057 (+4 ms)Region opened successfully at 1733267680057 2024-12-03T23:14:40,058 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegionServer(2236): Post open deploy tasks for test,hhh,1733267676090.256869bf1dc8aa33be79adb19e13d134., pid=73, masterSystemTime=1733267680039 2024-12-03T23:14:40,061 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegionServer(2266): Finished post open deploy task for test,hhh,1733267676090.256869bf1dc8aa33be79adb19e13d134. 2024-12-03T23:14:40,061 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] handler.AssignRegionHandler(153): Opened test,hhh,1733267676090.256869bf1dc8aa33be79adb19e13d134. 2024-12-03T23:14:40,061 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(132): Open test,sss,1733267676090.2b7da3736c6b257b43fe66ea13807bc6. 2024-12-03T23:14:40,061 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7752): Opening region: {ENCODED => 2b7da3736c6b257b43fe66ea13807bc6, NAME => 'test,sss,1733267676090.2b7da3736c6b257b43fe66ea13807bc6.', STARTKEY => 'sss', ENDKEY => 'ttt'} 2024-12-03T23:14:40,062 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 2b7da3736c6b257b43fe66ea13807bc6 2024-12-03T23:14:40,062 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(898): Instantiated test,sss,1733267676090.2b7da3736c6b257b43fe66ea13807bc6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T23:14:40,062 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7794): checking encryption for 2b7da3736c6b257b43fe66ea13807bc6 2024-12-03T23:14:40,062 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7797): checking classloading for 2b7da3736c6b257b43fe66ea13807bc6 2024-12-03T23:14:40,064 WARN [PEWorker-1 {}] procedure2.ProcedureTestingUtility(201): Set Kill before store update in rollback to: true 2024-12-03T23:14:40,064 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 256869bf1dc8aa33be79adb19e13d134, server=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,065 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1844): CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 256869bf1dc8aa33be79adb19e13d134, server=5f9d64419733,41167,1733267673270 java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T23:14:40,066 INFO [StoreOpener-2b7da3736c6b257b43fe66ea13807bc6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 2b7da3736c6b257b43fe66ea13807bc6 2024-12-03T23:14:40,069 INFO [StoreOpener-2b7da3736c6b257b43fe66ea13807bc6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2b7da3736c6b257b43fe66ea13807bc6 columnFamilyName family 2024-12-03T23:14:40,070 DEBUG [StoreOpener-2b7da3736c6b257b43fe66ea13807bc6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:14:40,070 INFO [StoreOpener-2b7da3736c6b257b43fe66ea13807bc6-1 {}] regionserver.HStore(327): Store=2b7da3736c6b257b43fe66ea13807bc6/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T23:14:40,072 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1038): replaying wal for 2b7da3736c6b257b43fe66ea13807bc6 2024-12-03T23:14:40,073 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/default/test/2b7da3736c6b257b43fe66ea13807bc6 2024-12-03T23:14:40,075 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/default/test/2b7da3736c6b257b43fe66ea13807bc6 2024-12-03T23:14:40,076 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1048): stopping wal replay for 2b7da3736c6b257b43fe66ea13807bc6 2024-12-03T23:14:40,076 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1060): Cleaning up temporary data for 2b7da3736c6b257b43fe66ea13807bc6 2024-12-03T23:14:40,076 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=true, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 256869bf1dc8aa33be79adb19e13d134, server=5f9d64419733,41167,1733267673270:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5f9d64419733,46833,1733267673070, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1845) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 4 more 2024-12-03T23:14:40,078 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-03T23:14:40,078 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=79, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 742c43a49ea3b763b8232c8e8c8966b3, server=5f9d64419733,44899,1733267673202 2024-12-03T23:14:40,078 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T23:14:40,078 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=79, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 742c43a49ea3b763b8232c8e8c8966b3, server=5f9d64419733,44899,1733267673202 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T23:14:40,079 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1093): writing seq id for 2b7da3736c6b257b43fe66ea13807bc6 2024-12-03T23:14:40,080 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1114): Opened 2b7da3736c6b257b43fe66ea13807bc6; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66468248, jitterRate=-0.00954592227935791}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T23:14:40,080 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2b7da3736c6b257b43fe66ea13807bc6 2024-12-03T23:14:40,080 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1006): Region open journal for 2b7da3736c6b257b43fe66ea13807bc6: Running coprocessor pre-open hook at 1733267680062Writing region info on filesystem at 1733267680062Initializing all the Stores at 1733267680065 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733267680065Cleaning up temporary data from old regions at 1733267680076 (+11 ms)Running coprocessor post-open hooks at 1733267680080 (+4 ms)Region opened successfully at 1733267680080 2024-12-03T23:14:40,081 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2236): Post open deploy tasks for test,sss,1733267676090.2b7da3736c6b257b43fe66ea13807bc6., pid=75, masterSystemTime=1733267680039 2024-12-03T23:14:40,084 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2266): Finished post open deploy task for test,sss,1733267676090.2b7da3736c6b257b43fe66ea13807bc6. 2024-12-03T23:14:40,084 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(153): Opened test,sss,1733267676090.2b7da3736c6b257b43fe66ea13807bc6. 2024-12-03T23:14:40,084 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(132): Open test,yyy,1733267676090.553e1b6e96399dd43d3b8f7cfad54903. 2024-12-03T23:14:40,084 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7752): Opening region: {ENCODED => 553e1b6e96399dd43d3b8f7cfad54903, NAME => 'test,yyy,1733267676090.553e1b6e96399dd43d3b8f7cfad54903.', STARTKEY => 'yyy', ENDKEY => 'zzz'} 2024-12-03T23:14:40,085 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 553e1b6e96399dd43d3b8f7cfad54903 2024-12-03T23:14:40,085 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(898): Instantiated test,yyy,1733267676090.553e1b6e96399dd43d3b8f7cfad54903.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T23:14:40,085 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7794): checking encryption for 553e1b6e96399dd43d3b8f7cfad54903 2024-12-03T23:14:40,085 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7797): checking classloading for 553e1b6e96399dd43d3b8f7cfad54903 2024-12-03T23:14:40,087 INFO [StoreOpener-553e1b6e96399dd43d3b8f7cfad54903-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 553e1b6e96399dd43d3b8f7cfad54903 2024-12-03T23:14:40,088 INFO [StoreOpener-553e1b6e96399dd43d3b8f7cfad54903-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 553e1b6e96399dd43d3b8f7cfad54903 columnFamilyName family 2024-12-03T23:14:40,088 DEBUG [StoreOpener-553e1b6e96399dd43d3b8f7cfad54903-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:14:40,089 INFO [StoreOpener-553e1b6e96399dd43d3b8f7cfad54903-1 {}] regionserver.HStore(327): Store=553e1b6e96399dd43d3b8f7cfad54903/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T23:14:40,089 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1038): replaying wal for 553e1b6e96399dd43d3b8f7cfad54903 2024-12-03T23:14:40,090 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/default/test/553e1b6e96399dd43d3b8f7cfad54903 2024-12-03T23:14:40,091 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/default/test/553e1b6e96399dd43d3b8f7cfad54903 2024-12-03T23:14:40,092 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1048): stopping wal replay for 553e1b6e96399dd43d3b8f7cfad54903 2024-12-03T23:14:40,092 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1060): Cleaning up temporary data for 553e1b6e96399dd43d3b8f7cfad54903 2024-12-03T23:14:40,094 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1093): writing seq id for 553e1b6e96399dd43d3b8f7cfad54903 2024-12-03T23:14:40,095 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1114): Opened 553e1b6e96399dd43d3b8f7cfad54903; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61998695, jitterRate=-0.07614745199680328}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T23:14:40,095 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 553e1b6e96399dd43d3b8f7cfad54903 2024-12-03T23:14:40,095 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1006): Region open journal for 553e1b6e96399dd43d3b8f7cfad54903: Running coprocessor pre-open hook at 1733267680085Writing region info on filesystem at 1733267680085Initializing all the Stores at 1733267680086 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733267680086Cleaning up temporary data from old regions at 1733267680092 (+6 ms)Running coprocessor post-open hooks at 1733267680095 (+3 ms)Region opened successfully at 1733267680095 2024-12-03T23:14:40,096 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2236): Post open deploy tasks for test,yyy,1733267676090.553e1b6e96399dd43d3b8f7cfad54903., pid=77, masterSystemTime=1733267680039 2024-12-03T23:14:40,099 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2266): Finished post open deploy task for test,yyy,1733267676090.553e1b6e96399dd43d3b8f7cfad54903. 2024-12-03T23:14:40,099 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(153): Opened test,yyy,1733267676090.553e1b6e96399dd43d3b8f7cfad54903. 2024-12-03T23:14:40,099 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] handler.AssignRegionHandler(132): Open test,mmm,1733267676090.4e6706440af7674db97bb48a05858115. 2024-12-03T23:14:40,099 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(7752): Opening region: {ENCODED => 4e6706440af7674db97bb48a05858115, NAME => 'test,mmm,1733267676090.4e6706440af7674db97bb48a05858115.', STARTKEY => 'mmm', ENDKEY => 'nnn'} 2024-12-03T23:14:40,099 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 4e6706440af7674db97bb48a05858115 2024-12-03T23:14:40,099 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(898): Instantiated test,mmm,1733267676090.4e6706440af7674db97bb48a05858115.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T23:14:40,099 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(7794): checking encryption for 4e6706440af7674db97bb48a05858115 2024-12-03T23:14:40,100 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(7797): checking classloading for 4e6706440af7674db97bb48a05858115 2024-12-03T23:14:40,101 INFO [StoreOpener-4e6706440af7674db97bb48a05858115-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 4e6706440af7674db97bb48a05858115 2024-12-03T23:14:40,102 INFO [StoreOpener-4e6706440af7674db97bb48a05858115-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4e6706440af7674db97bb48a05858115 columnFamilyName family 2024-12-03T23:14:40,102 DEBUG [StoreOpener-4e6706440af7674db97bb48a05858115-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:14:40,102 INFO [StoreOpener-4e6706440af7674db97bb48a05858115-1 {}] regionserver.HStore(327): Store=4e6706440af7674db97bb48a05858115/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T23:14:40,102 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1038): replaying wal for 4e6706440af7674db97bb48a05858115 2024-12-03T23:14:40,103 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/default/test/4e6706440af7674db97bb48a05858115 2024-12-03T23:14:40,105 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/default/test/4e6706440af7674db97bb48a05858115 2024-12-03T23:14:40,105 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1048): stopping wal replay for 4e6706440af7674db97bb48a05858115 2024-12-03T23:14:40,105 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1060): Cleaning up temporary data for 4e6706440af7674db97bb48a05858115 2024-12-03T23:14:40,108 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1093): writing seq id for 4e6706440af7674db97bb48a05858115 2024-12-03T23:14:40,109 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1114): Opened 4e6706440af7674db97bb48a05858115; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67524001, jitterRate=0.0061860233545303345}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T23:14:40,109 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4e6706440af7674db97bb48a05858115 2024-12-03T23:14:40,109 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1006): Region open journal for 4e6706440af7674db97bb48a05858115: Running coprocessor pre-open hook at 1733267680100Writing region info on filesystem at 1733267680100Initializing all the Stores at 1733267680100Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733267680101 (+1 ms)Cleaning up temporary data from old regions at 1733267680105 (+4 ms)Running coprocessor post-open hooks at 1733267680109 (+4 ms)Region opened successfully at 1733267680109 2024-12-03T23:14:40,110 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegionServer(2236): Post open deploy tasks for test,mmm,1733267676090.4e6706440af7674db97bb48a05858115., pid=72, masterSystemTime=1733267680039 2024-12-03T23:14:40,112 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-03T23:14:40,112 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T23:14:40,112 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-03T23:14:40,112 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-03T23:14:40,113 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegionServer(2266): Finished post open deploy task for test,mmm,1733267676090.4e6706440af7674db97bb48a05858115. 2024-12-03T23:14:40,113 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] handler.AssignRegionHandler(153): Opened test,mmm,1733267676090.4e6706440af7674db97bb48a05858115. 2024-12-03T23:14:40,113 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-03T23:14:40,113 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-03T23:14:40,113 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-03T23:14:40,113 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-03T23:14:40,114 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-03T23:14:40,119 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-03T23:14:40,119 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T23:14:40,120 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=64, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-03T23:14:40,120 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T23:14:40,120 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-03T23:14:40,120 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T23:14:40,120 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-03T23:14:40,120 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T23:14:40,120 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-03T23:14:40,120 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T23:14:40,120 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-03T23:14:40,120 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T23:14:40,120 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-03T23:14:40,120 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T23:14:40,120 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-03T23:14:40,120 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T23:14:40,120 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-03T23:14:40,120 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-03T23:14:40,120 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-03T23:14:40,120 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T23:14:40,120 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-03T23:14:40,120 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-03T23:14:40,120 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-03T23:14:40,121 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-03T23:14:40,122 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-03T23:14:40,123 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44117 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T23:14:40,127 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 256869bf1dc8aa33be79adb19e13d134, server=5f9d64419733,41167,1733267673270:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5f9d64419733,46833,1733267673070, splitWal=true, meta=true 2024-12-03T23:14:40,127 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5f9d64419733%2C46833%2C1733267673070.meta.1733267675470.meta, worker=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,127 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5f9d64419733%2C46833%2C1733267673070.meta.1733267675470.meta, worker=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,127 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-03T23:14:40,127 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,127 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5f9d64419733%2C46833%2C1733267673070.1733267675041, worker=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,127 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5f9d64419733%2C46833%2C1733267673070.1733267675041, worker=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,127 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=23a123891f32f583441cd7569a6de1f2, ASSIGN 2024-12-03T23:14:40,127 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b7da3736c6b257b43fe66ea13807bc6, ASSIGN 2024-12-03T23:14:40,127 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e6706440af7674db97bb48a05858115, ASSIGN 2024-12-03T23:14:40,127 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=256869bf1dc8aa33be79adb19e13d134, ASSIGN 2024-12-03T23:14:40,127 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=742c43a49ea3b763b8232c8e8c8966b3, ASSIGN 2024-12-03T23:14:40,127 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=553e1b6e96399dd43d3b8f7cfad54903, ASSIGN 2024-12-03T23:14:40,127 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=384b2e2791a8bf699cb7996e1aa7469c, ASSIGN 2024-12-03T23:14:40,127 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6d9981bd2d541debb970a37ac121e7ed, ASSIGN 2024-12-03T23:14:40,127 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4e6706440af7674db97bb48a05858115, server=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,127 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=67, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 256869bf1dc8aa33be79adb19e13d134, server=5f9d64419733,41167,1733267673270:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 256869bf1dc8aa33be79adb19e13d134, server=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,127 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 23a123891f32f583441cd7569a6de1f2, server=5f9d64419733,44899,1733267673202 2024-12-03T23:14:40,127 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2b7da3736c6b257b43fe66ea13807bc6, server=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,127 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6d9981bd2d541debb970a37ac121e7ed, server=5f9d64419733,44899,1733267673202 2024-12-03T23:14:40,127 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 553e1b6e96399dd43d3b8f7cfad54903, server=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,127 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 384b2e2791a8bf699cb7996e1aa7469c, server=5f9d64419733,44899,1733267673202 2024-12-03T23:14:40,127 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 742c43a49ea3b763b8232c8e8c8966b3, server=5f9d64419733,44899,1733267673202 2024-12-03T23:14:40,128 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 256869bf1dc8aa33be79adb19e13d134, server=5f9d64419733,41167,1733267673270:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5f9d64419733,46833,1733267673070, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,128 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5f9d64419733%2C46833%2C1733267673070.meta.1733267675470.meta, worker=5f9d64419733,41167,1733267673270 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,128 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5f9d64419733%2C46833%2C1733267673070.meta.1733267675470.meta, worker=5f9d64419733,41167,1733267673270 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,128 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,128 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=5f9d64419733,41167,1733267673270 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,128 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5f9d64419733%2C46833%2C1733267673070.1733267675041, worker=5f9d64419733,41167,1733267673270 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,128 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5f9d64419733%2C46833%2C1733267673070.1733267675041, worker=5f9d64419733,41167,1733267673270 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,128 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=23a123891f32f583441cd7569a6de1f2, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,128 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=23a123891f32f583441cd7569a6de1f2, ASSIGN 2024-12-03T23:14:40,128 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b7da3736c6b257b43fe66ea13807bc6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,128 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b7da3736c6b257b43fe66ea13807bc6, ASSIGN 2024-12-03T23:14:40,128 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e6706440af7674db97bb48a05858115, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,128 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e6706440af7674db97bb48a05858115, ASSIGN 2024-12-03T23:14:40,128 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=256869bf1dc8aa33be79adb19e13d134, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,128 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=256869bf1dc8aa33be79adb19e13d134, ASSIGN 2024-12-03T23:14:40,128 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=742c43a49ea3b763b8232c8e8c8966b3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,128 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=742c43a49ea3b763b8232c8e8c8966b3, ASSIGN 2024-12-03T23:14:40,128 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=553e1b6e96399dd43d3b8f7cfad54903, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,128 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=553e1b6e96399dd43d3b8f7cfad54903, ASSIGN 2024-12-03T23:14:40,128 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=384b2e2791a8bf699cb7996e1aa7469c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,128 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=384b2e2791a8bf699cb7996e1aa7469c, ASSIGN 2024-12-03T23:14:40,128 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6d9981bd2d541debb970a37ac121e7ed, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,128 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6d9981bd2d541debb970a37ac121e7ed, ASSIGN 2024-12-03T23:14:40,128 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4e6706440af7674db97bb48a05858115, server=5f9d64419733,41167,1733267673270 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,129 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=67, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 256869bf1dc8aa33be79adb19e13d134, server=5f9d64419733,41167,1733267673270:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 256869bf1dc8aa33be79adb19e13d134, server=5f9d64419733,41167,1733267673270 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,129 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 23a123891f32f583441cd7569a6de1f2, server=5f9d64419733,44899,1733267673202 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,129 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2b7da3736c6b257b43fe66ea13807bc6, server=5f9d64419733,41167,1733267673270 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,129 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6d9981bd2d541debb970a37ac121e7ed, server=5f9d64419733,44899,1733267673202 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,129 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 553e1b6e96399dd43d3b8f7cfad54903, server=5f9d64419733,41167,1733267673270 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,129 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 384b2e2791a8bf699cb7996e1aa7469c, server=5f9d64419733,44899,1733267673202 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,129 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 742c43a49ea3b763b8232c8e8c8966b3, server=5f9d64419733,44899,1733267673202 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,129 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 15 msec 2024-12-03T23:14:40,129 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44117 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T23:14:40,131 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=61 2024-12-03T23:14:40,131 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=5f9d64419733,41167,1733267673270, table=hbase:meta, region=1588230740 2024-12-03T23:14:40,131 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 5f9d64419733,41167,1733267673270, state=OPEN 2024-12-03T23:14:40,133 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41167-0x1017a9bb4a60003, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T23:14:40,133 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44899-0x1017a9bb4a60002, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T23:14:40,133 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44117-0x1017a9bb4a60000, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T23:14:40,133 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T23:14:40,133 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T23:14:40,133 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T23:14:40,135 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=23a123891f32f583441cd7569a6de1f2, ASSIGN to state=OFFLINE, location=null, table=test, region=23a123891f32f583441cd7569a6de1f2 to restore RIT 2024-12-03T23:14:40,135 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b7da3736c6b257b43fe66ea13807bc6, ASSIGN to state=OFFLINE, location=null, table=test, region=2b7da3736c6b257b43fe66ea13807bc6 to restore RIT 2024-12-03T23:14:40,135 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e6706440af7674db97bb48a05858115, ASSIGN to state=OFFLINE, location=null, table=test, region=4e6706440af7674db97bb48a05858115 to restore RIT 2024-12-03T23:14:40,135 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=256869bf1dc8aa33be79adb19e13d134, ASSIGN to state=OFFLINE, location=null, table=test, region=256869bf1dc8aa33be79adb19e13d134 to restore RIT 2024-12-03T23:14:40,135 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=742c43a49ea3b763b8232c8e8c8966b3, ASSIGN to state=OFFLINE, location=null, table=test, region=742c43a49ea3b763b8232c8e8c8966b3 to restore RIT 2024-12-03T23:14:40,135 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=553e1b6e96399dd43d3b8f7cfad54903, ASSIGN to state=OFFLINE, location=null, table=test, region=553e1b6e96399dd43d3b8f7cfad54903 to restore RIT 2024-12-03T23:14:40,135 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=384b2e2791a8bf699cb7996e1aa7469c, ASSIGN to state=OFFLINE, location=null, table=test, region=384b2e2791a8bf699cb7996e1aa7469c to restore RIT 2024-12-03T23:14:40,135 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6d9981bd2d541debb970a37ac121e7ed, ASSIGN to state=OFFLINE, location=null, table=test, region=6d9981bd2d541debb970a37ac121e7ed to restore RIT 2024-12-03T23:14:40,136 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733267710136 2024-12-03T23:14:40,136 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-03T23:14:40,136 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 256869bf1dc8aa33be79adb19e13d134, server=5f9d64419733,41167,1733267673270:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5f9d64419733,46833,1733267673070, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T23:14:40,138 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-03T23:14:40,140 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-03T23:14:40,140 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=78, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 384b2e2791a8bf699cb7996e1aa7469c, server=5f9d64419733,44899,1733267673202 2024-12-03T23:14:40,140 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T23:14:40,140 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=78, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 384b2e2791a8bf699cb7996e1aa7469c, server=5f9d64419733,44899,1733267673202 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T23:14:40,146 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] handler.AssignRegionHandler(132): Open test,jjj,1733267676090.384b2e2791a8bf699cb7996e1aa7469c. 2024-12-03T23:14:40,146 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7752): Opening region: {ENCODED => 384b2e2791a8bf699cb7996e1aa7469c, NAME => 'test,jjj,1733267676090.384b2e2791a8bf699cb7996e1aa7469c.', STARTKEY => 'jjj', ENDKEY => 'kkk'} 2024-12-03T23:14:40,146 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 384b2e2791a8bf699cb7996e1aa7469c 2024-12-03T23:14:40,146 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(898): Instantiated test,jjj,1733267676090.384b2e2791a8bf699cb7996e1aa7469c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T23:14:40,146 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7794): checking encryption for 384b2e2791a8bf699cb7996e1aa7469c 2024-12-03T23:14:40,147 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7797): checking classloading for 384b2e2791a8bf699cb7996e1aa7469c 2024-12-03T23:14:40,149 INFO [StoreOpener-384b2e2791a8bf699cb7996e1aa7469c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 384b2e2791a8bf699cb7996e1aa7469c 2024-12-03T23:14:40,150 INFO [StoreOpener-384b2e2791a8bf699cb7996e1aa7469c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 384b2e2791a8bf699cb7996e1aa7469c columnFamilyName family 2024-12-03T23:14:40,150 DEBUG [StoreOpener-384b2e2791a8bf699cb7996e1aa7469c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:14:40,151 INFO [StoreOpener-384b2e2791a8bf699cb7996e1aa7469c-1 {}] regionserver.HStore(327): Store=384b2e2791a8bf699cb7996e1aa7469c/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T23:14:40,151 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1038): replaying wal for 384b2e2791a8bf699cb7996e1aa7469c 2024-12-03T23:14:40,152 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/default/test/384b2e2791a8bf699cb7996e1aa7469c 2024-12-03T23:14:40,153 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/default/test/384b2e2791a8bf699cb7996e1aa7469c 2024-12-03T23:14:40,154 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1048): stopping wal replay for 384b2e2791a8bf699cb7996e1aa7469c 2024-12-03T23:14:40,154 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1060): Cleaning up temporary data for 384b2e2791a8bf699cb7996e1aa7469c 2024-12-03T23:14:40,156 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1093): writing seq id for 384b2e2791a8bf699cb7996e1aa7469c 2024-12-03T23:14:40,158 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1114): Opened 384b2e2791a8bf699cb7996e1aa7469c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71459355, jitterRate=0.06482736766338348}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T23:14:40,158 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 384b2e2791a8bf699cb7996e1aa7469c 2024-12-03T23:14:40,158 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1006): Region open journal for 384b2e2791a8bf699cb7996e1aa7469c: Running coprocessor pre-open hook at 1733267680147Writing region info on filesystem at 1733267680147Initializing all the Stores at 1733267680148 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733267680148Cleaning up temporary data from old regions at 1733267680154 (+6 ms)Running coprocessor post-open hooks at 1733267680158 (+4 ms)Region opened successfully at 1733267680158 2024-12-03T23:14:40,159 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionServer(2236): Post open deploy tasks for test,jjj,1733267676090.384b2e2791a8bf699cb7996e1aa7469c., pid=78, masterSystemTime=1733267680140 2024-12-03T23:14:40,161 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44117 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.PleaseHoldException: Master not fully online; hbase:meta=false, metaLoaded=false at org.apache.hadoop.hbase.master.assignment.AssignmentManager.checkMetaLoaded(AssignmentManager.java:1924) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.AssignmentManager.updateRegionTransition(AssignmentManager.java:1288) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.AssignmentManager.reportRegionStateTransition(AssignmentManager.java:1214) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.AssignmentManager.reportRegionStateTransition(AssignmentManager.java:1264) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1867) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T23:14:40,165 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44117 {}] ipc.CallRunner(138): callId: 60 service: RegionServerStatusService methodName: ReportRegionStateTransition size: 126 connection: 172.17.0.2:43099 deadline: 1733267690160, exception=org.apache.hadoop.hbase.PleaseHoldException: Master not fully online; hbase:meta=false, metaLoaded=false 2024-12-03T23:14:40,167 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionServer(2367): Failed report transition server { host_name: "5f9d64419733" port: 44899 start_code: 1733267673202 } transition { transition_code: OPENED region_info { region_id: 1733267676090 table_name { namespace: "default" qualifier: "test" } start_key: "jjj" end_key: "kkk" offline: false split: false replica_id: 0 } open_seq_num: 2 proc_id: 78 initiating_master_active_time: 1733267673415 }; retry (#0) after 1000ms delay (Master is coming online...). org.apache.hadoop.hbase.PleaseHoldException: org.apache.hadoop.hbase.PleaseHoldException: Master not fully online; hbase:meta=false, metaLoaded=false at org.apache.hadoop.hbase.master.assignment.AssignmentManager.checkMetaLoaded(AssignmentManager.java:1924) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.updateRegionTransition(AssignmentManager.java:1288) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.reportRegionStateTransition(AssignmentManager.java:1214) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.reportRegionStateTransition(AssignmentManager.java:1264) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1867) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:403) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.getRemoteException(ProtobufUtil.java:381) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportRegionStateTransition(HRegionServer.java:2358) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2257) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.PleaseHoldException: Master not fully online; hbase:meta=false, metaLoaded=false at org.apache.hadoop.hbase.master.assignment.AssignmentManager.checkMetaLoaded(AssignmentManager.java:1924) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.updateRegionTransition(AssignmentManager.java:1288) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.reportRegionStateTransition(AssignmentManager.java:1214) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.reportRegionStateTransition(AssignmentManager.java:1264) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1867) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-12-03T23:14:40,169 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d9981bd2d541debb970a37ac121e7ed, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,169 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f71f90ef2d2f59e885f2ad6ca1a99596, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,170 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e35e57b6f6cbff01d3e782fbf6efd1f, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,170 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=55eff67791415c655cef877182b9bbe1, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,170 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=212a9963c9cddede26a4eb521cff5254, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,171 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f514b27fbd5c6ef451e11852713ea906, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,171 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=73cd6e72f89a0d14d1d979f76c9c07a4, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,171 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=256869bf1dc8aa33be79adb19e13d134, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,172 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2db7cef12ba09a51b0a7204fc81deeec, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,172 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=384b2e2791a8bf699cb7996e1aa7469c, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,172 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5ddac1aa9a6f0af338b83490d6d952c2, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,173 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1d026a7a0318e7deb79fa16b1262ae2c, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,173 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4e6706440af7674db97bb48a05858115, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,173 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f00ca5d778884039f3248393a629dcfc, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,174 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5e6925efcdb64a8de08b4c4bbf3a3d2e, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,174 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=742c43a49ea3b763b8232c8e8c8966b3, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,174 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=10beaf4047510d958e4a559194242a57, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,175 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=89a698454f10c9fb6cfc108c35d8415a, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,175 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b7da3736c6b257b43fe66ea13807bc6, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,175 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a6de975926600fad463e13238b2ac8a9, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,176 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e5e4ecc042f874ec5567f7e86548256f, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,176 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23a123891f32f583441cd7569a6de1f2, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,177 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0f09ea7cbe3ceec5232d11c21a512a4, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,177 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e96b12de28d6c3047803f54a3b027459, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,177 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=553e1b6e96399dd43d3b8f7cfad54903, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,178 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=db9924063978f5acb21b073c5d94a7ad, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,178 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-03T23:14:40,178 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733267740178 2024-12-03T23:14:40,178 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733267800178 2024-12-03T23:14:40,178 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 41 msec 2024-12-03T23:14:40,178 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-03T23:14:40,178 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T23:14:40,178 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-03T23:14:40,178 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-03T23:14:40,179 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-03T23:14:40,179 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-03T23:14:40,179 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-03T23:14:40,179 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-03T23:14:40,180 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-03T23:14:40,186 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-03T23:14:40,186 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T23:14:40,186 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=64, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-03T23:14:40,186 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T23:14:40,186 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-03T23:14:40,186 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T23:14:40,186 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-03T23:14:40,186 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T23:14:40,186 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-03T23:14:40,186 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T23:14:40,186 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T23:14:40,186 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-03T23:14:40,186 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T23:14:40,186 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-03T23:14:40,186 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T23:14:40,186 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-03T23:14:40,187 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-03T23:14:40,187 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-03T23:14:40,187 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T23:14:40,187 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-03T23:14:40,187 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-03T23:14:40,187 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-03T23:14:40,187 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-03T23:14:40,188 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-03T23:14:40,192 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 256869bf1dc8aa33be79adb19e13d134, server=5f9d64419733,41167,1733267673270:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5f9d64419733,46833,1733267673070, splitWal=true, meta=true 2024-12-03T23:14:40,193 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5f9d64419733%2C46833%2C1733267673070.meta.1733267675470.meta, worker=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,193 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5f9d64419733%2C46833%2C1733267673070.meta.1733267675470.meta, worker=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,193 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-03T23:14:40,193 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,193 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5f9d64419733%2C46833%2C1733267673070.1733267675041, worker=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,193 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5f9d64419733%2C46833%2C1733267673070.1733267675041, worker=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,193 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=23a123891f32f583441cd7569a6de1f2, ASSIGN 2024-12-03T23:14:40,193 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b7da3736c6b257b43fe66ea13807bc6, ASSIGN 2024-12-03T23:14:40,193 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e6706440af7674db97bb48a05858115, ASSIGN 2024-12-03T23:14:40,193 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=256869bf1dc8aa33be79adb19e13d134, ASSIGN 2024-12-03T23:14:40,193 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=742c43a49ea3b763b8232c8e8c8966b3, ASSIGN 2024-12-03T23:14:40,193 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=553e1b6e96399dd43d3b8f7cfad54903, ASSIGN 2024-12-03T23:14:40,193 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=384b2e2791a8bf699cb7996e1aa7469c, ASSIGN 2024-12-03T23:14:40,193 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6d9981bd2d541debb970a37ac121e7ed, ASSIGN 2024-12-03T23:14:40,193 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4e6706440af7674db97bb48a05858115, server=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,193 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=67, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 256869bf1dc8aa33be79adb19e13d134, server=5f9d64419733,41167,1733267673270:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 256869bf1dc8aa33be79adb19e13d134, server=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,193 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 23a123891f32f583441cd7569a6de1f2, server=5f9d64419733,44899,1733267673202 2024-12-03T23:14:40,193 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2b7da3736c6b257b43fe66ea13807bc6, server=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,193 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6d9981bd2d541debb970a37ac121e7ed, server=5f9d64419733,44899,1733267673202 2024-12-03T23:14:40,193 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 553e1b6e96399dd43d3b8f7cfad54903, server=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,193 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 384b2e2791a8bf699cb7996e1aa7469c, server=5f9d64419733,44899,1733267673202 2024-12-03T23:14:40,193 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 256869bf1dc8aa33be79adb19e13d134, server=5f9d64419733,41167,1733267673270:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5f9d64419733,46833,1733267673070, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,194 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5f9d64419733%2C46833%2C1733267673070.meta.1733267675470.meta, worker=5f9d64419733,41167,1733267673270 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,194 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5f9d64419733%2C46833%2C1733267673070.meta.1733267675470.meta, worker=5f9d64419733,41167,1733267673270 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,194 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,194 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=5f9d64419733,41167,1733267673270 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,194 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5f9d64419733%2C46833%2C1733267673070.1733267675041, worker=5f9d64419733,41167,1733267673270 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,194 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5f9d64419733%2C46833%2C1733267673070.1733267675041, worker=5f9d64419733,41167,1733267673270 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,194 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=23a123891f32f583441cd7569a6de1f2, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,194 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=23a123891f32f583441cd7569a6de1f2, ASSIGN 2024-12-03T23:14:40,194 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b7da3736c6b257b43fe66ea13807bc6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,194 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b7da3736c6b257b43fe66ea13807bc6, ASSIGN 2024-12-03T23:14:40,194 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e6706440af7674db97bb48a05858115, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,194 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e6706440af7674db97bb48a05858115, ASSIGN 2024-12-03T23:14:40,194 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=256869bf1dc8aa33be79adb19e13d134, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,194 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=256869bf1dc8aa33be79adb19e13d134, ASSIGN 2024-12-03T23:14:40,194 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=742c43a49ea3b763b8232c8e8c8966b3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,194 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=742c43a49ea3b763b8232c8e8c8966b3, ASSIGN 2024-12-03T23:14:40,194 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=553e1b6e96399dd43d3b8f7cfad54903, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,194 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=553e1b6e96399dd43d3b8f7cfad54903, ASSIGN 2024-12-03T23:14:40,194 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=384b2e2791a8bf699cb7996e1aa7469c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,194 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=384b2e2791a8bf699cb7996e1aa7469c, ASSIGN 2024-12-03T23:14:40,194 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6d9981bd2d541debb970a37ac121e7ed, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,194 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6d9981bd2d541debb970a37ac121e7ed, ASSIGN 2024-12-03T23:14:40,194 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4e6706440af7674db97bb48a05858115, server=5f9d64419733,41167,1733267673270 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,194 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=67, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 256869bf1dc8aa33be79adb19e13d134, server=5f9d64419733,41167,1733267673270:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 256869bf1dc8aa33be79adb19e13d134, server=5f9d64419733,41167,1733267673270 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,194 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 23a123891f32f583441cd7569a6de1f2, server=5f9d64419733,44899,1733267673202 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,194 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2b7da3736c6b257b43fe66ea13807bc6, server=5f9d64419733,41167,1733267673270 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,194 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6d9981bd2d541debb970a37ac121e7ed, server=5f9d64419733,44899,1733267673202 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,195 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 553e1b6e96399dd43d3b8f7cfad54903, server=5f9d64419733,41167,1733267673270 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,195 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 384b2e2791a8bf699cb7996e1aa7469c, server=5f9d64419733,44899,1733267673202 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,195 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 14 msec 2024-12-03T23:14:40,196 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=61 2024-12-03T23:14:40,196 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=5f9d64419733,41167,1733267673270, table=hbase:meta, region=1588230740 2024-12-03T23:14:40,197 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 5f9d64419733,41167,1733267673270, state=OPEN 2024-12-03T23:14:40,199 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44899-0x1017a9bb4a60002, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T23:14:40,199 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44117-0x1017a9bb4a60000, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T23:14:40,199 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41167-0x1017a9bb4a60003, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T23:14:40,199 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T23:14:40,199 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T23:14:40,199 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T23:14:40,199 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=23a123891f32f583441cd7569a6de1f2, ASSIGN to state=OFFLINE, location=null, table=test, region=23a123891f32f583441cd7569a6de1f2 to restore RIT 2024-12-03T23:14:40,200 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b7da3736c6b257b43fe66ea13807bc6, ASSIGN to state=OFFLINE, location=null, table=test, region=2b7da3736c6b257b43fe66ea13807bc6 to restore RIT 2024-12-03T23:14:40,200 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e6706440af7674db97bb48a05858115, ASSIGN to state=OFFLINE, location=null, table=test, region=4e6706440af7674db97bb48a05858115 to restore RIT 2024-12-03T23:14:40,200 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=256869bf1dc8aa33be79adb19e13d134, ASSIGN to state=OFFLINE, location=null, table=test, region=256869bf1dc8aa33be79adb19e13d134 to restore RIT 2024-12-03T23:14:40,200 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=742c43a49ea3b763b8232c8e8c8966b3, ASSIGN to state=OFFLINE, location=null, table=test, region=742c43a49ea3b763b8232c8e8c8966b3 to restore RIT 2024-12-03T23:14:40,200 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=553e1b6e96399dd43d3b8f7cfad54903, ASSIGN to state=OFFLINE, location=null, table=test, region=553e1b6e96399dd43d3b8f7cfad54903 to restore RIT 2024-12-03T23:14:40,200 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=384b2e2791a8bf699cb7996e1aa7469c, ASSIGN to state=OFFLINE, location=null, table=test, region=384b2e2791a8bf699cb7996e1aa7469c to restore RIT 2024-12-03T23:14:40,200 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6d9981bd2d541debb970a37ac121e7ed, ASSIGN to state=OFFLINE, location=null, table=test, region=6d9981bd2d541debb970a37ac121e7ed to restore RIT 2024-12-03T23:14:40,200 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733267710200 2024-12-03T23:14:40,200 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-03T23:14:40,200 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 256869bf1dc8aa33be79adb19e13d134, server=5f9d64419733,41167,1733267673270:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5f9d64419733,46833,1733267673070, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T23:14:40,202 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-03T23:14:40,203 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-03T23:14:40,203 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=77, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 553e1b6e96399dd43d3b8f7cfad54903, server=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,203 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T23:14:40,204 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=77, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 553e1b6e96399dd43d3b8f7cfad54903, server=5f9d64419733,41167,1733267673270 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T23:14:40,210 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d9981bd2d541debb970a37ac121e7ed, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,211 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f71f90ef2d2f59e885f2ad6ca1a99596, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,211 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e35e57b6f6cbff01d3e782fbf6efd1f, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,211 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=55eff67791415c655cef877182b9bbe1, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,211 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=212a9963c9cddede26a4eb521cff5254, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,212 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f514b27fbd5c6ef451e11852713ea906, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,212 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=73cd6e72f89a0d14d1d979f76c9c07a4, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,212 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=256869bf1dc8aa33be79adb19e13d134, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,212 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2db7cef12ba09a51b0a7204fc81deeec, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,213 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=384b2e2791a8bf699cb7996e1aa7469c, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,213 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5ddac1aa9a6f0af338b83490d6d952c2, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,213 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1d026a7a0318e7deb79fa16b1262ae2c, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,213 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4e6706440af7674db97bb48a05858115, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,214 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f00ca5d778884039f3248393a629dcfc, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,214 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5e6925efcdb64a8de08b4c4bbf3a3d2e, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,214 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=742c43a49ea3b763b8232c8e8c8966b3, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,214 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=10beaf4047510d958e4a559194242a57, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,214 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=89a698454f10c9fb6cfc108c35d8415a, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,215 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b7da3736c6b257b43fe66ea13807bc6, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,215 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a6de975926600fad463e13238b2ac8a9, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,215 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e5e4ecc042f874ec5567f7e86548256f, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,215 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23a123891f32f583441cd7569a6de1f2, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,216 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0f09ea7cbe3ceec5232d11c21a512a4, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,216 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e96b12de28d6c3047803f54a3b027459, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,216 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=553e1b6e96399dd43d3b8f7cfad54903, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,216 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=db9924063978f5acb21b073c5d94a7ad, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,216 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-03T23:14:40,216 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733267740216 2024-12-03T23:14:40,216 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733267800216 2024-12-03T23:14:40,217 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 16 msec 2024-12-03T23:14:40,217 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-03T23:14:40,217 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T23:14:40,217 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-03T23:14:40,217 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-03T23:14:40,217 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-03T23:14:40,217 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-03T23:14:40,217 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-03T23:14:40,218 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-03T23:14:40,219 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-03T23:14:40,223 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-03T23:14:40,224 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T23:14:40,224 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=64, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-03T23:14:40,224 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T23:14:40,224 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-03T23:14:40,224 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T23:14:40,224 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-03T23:14:40,224 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T23:14:40,224 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-03T23:14:40,224 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T23:14:40,224 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T23:14:40,224 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-03T23:14:40,224 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T23:14:40,224 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T23:14:40,224 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-03T23:14:40,224 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-03T23:14:40,224 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-03T23:14:40,224 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T23:14:40,224 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-03T23:14:40,224 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-03T23:14:40,224 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-03T23:14:40,225 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-03T23:14:40,226 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-03T23:14:40,227 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44117 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T23:14:40,229 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 256869bf1dc8aa33be79adb19e13d134, server=5f9d64419733,41167,1733267673270:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5f9d64419733,46833,1733267673070, splitWal=true, meta=true 2024-12-03T23:14:40,229 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5f9d64419733%2C46833%2C1733267673070.meta.1733267675470.meta, worker=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,229 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5f9d64419733%2C46833%2C1733267673070.meta.1733267675470.meta, worker=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,229 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-03T23:14:40,229 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,229 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5f9d64419733%2C46833%2C1733267673070.1733267675041, worker=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,229 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5f9d64419733%2C46833%2C1733267673070.1733267675041, worker=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,229 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=23a123891f32f583441cd7569a6de1f2, ASSIGN 2024-12-03T23:14:40,229 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b7da3736c6b257b43fe66ea13807bc6, ASSIGN 2024-12-03T23:14:40,229 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e6706440af7674db97bb48a05858115, ASSIGN 2024-12-03T23:14:40,229 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=256869bf1dc8aa33be79adb19e13d134, ASSIGN 2024-12-03T23:14:40,229 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=742c43a49ea3b763b8232c8e8c8966b3, ASSIGN 2024-12-03T23:14:40,229 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=553e1b6e96399dd43d3b8f7cfad54903, ASSIGN 2024-12-03T23:14:40,229 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=384b2e2791a8bf699cb7996e1aa7469c, ASSIGN 2024-12-03T23:14:40,229 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6d9981bd2d541debb970a37ac121e7ed, ASSIGN 2024-12-03T23:14:40,229 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4e6706440af7674db97bb48a05858115, server=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,229 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=67, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 256869bf1dc8aa33be79adb19e13d134, server=5f9d64419733,41167,1733267673270:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 256869bf1dc8aa33be79adb19e13d134, server=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,229 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 23a123891f32f583441cd7569a6de1f2, server=5f9d64419733,44899,1733267673202 2024-12-03T23:14:40,229 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2b7da3736c6b257b43fe66ea13807bc6, server=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,229 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6d9981bd2d541debb970a37ac121e7ed, server=5f9d64419733,44899,1733267673202 2024-12-03T23:14:40,229 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 553e1b6e96399dd43d3b8f7cfad54903, server=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,230 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 256869bf1dc8aa33be79adb19e13d134, server=5f9d64419733,41167,1733267673270:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5f9d64419733,46833,1733267673070, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,230 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5f9d64419733%2C46833%2C1733267673070.meta.1733267675470.meta, worker=5f9d64419733,41167,1733267673270 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,230 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5f9d64419733%2C46833%2C1733267673070.meta.1733267675470.meta, worker=5f9d64419733,41167,1733267673270 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,230 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,230 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=5f9d64419733,41167,1733267673270 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,230 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5f9d64419733%2C46833%2C1733267673070.1733267675041, worker=5f9d64419733,41167,1733267673270 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,230 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5f9d64419733%2C46833%2C1733267673070.1733267675041, worker=5f9d64419733,41167,1733267673270 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,230 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=23a123891f32f583441cd7569a6de1f2, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,230 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=23a123891f32f583441cd7569a6de1f2, ASSIGN 2024-12-03T23:14:40,230 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b7da3736c6b257b43fe66ea13807bc6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,230 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b7da3736c6b257b43fe66ea13807bc6, ASSIGN 2024-12-03T23:14:40,230 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e6706440af7674db97bb48a05858115, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,230 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e6706440af7674db97bb48a05858115, ASSIGN 2024-12-03T23:14:40,230 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=256869bf1dc8aa33be79adb19e13d134, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,230 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=256869bf1dc8aa33be79adb19e13d134, ASSIGN 2024-12-03T23:14:40,230 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=742c43a49ea3b763b8232c8e8c8966b3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,230 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=742c43a49ea3b763b8232c8e8c8966b3, ASSIGN 2024-12-03T23:14:40,230 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=553e1b6e96399dd43d3b8f7cfad54903, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,230 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=553e1b6e96399dd43d3b8f7cfad54903, ASSIGN 2024-12-03T23:14:40,230 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=384b2e2791a8bf699cb7996e1aa7469c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,230 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=384b2e2791a8bf699cb7996e1aa7469c, ASSIGN 2024-12-03T23:14:40,230 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6d9981bd2d541debb970a37ac121e7ed, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,230 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6d9981bd2d541debb970a37ac121e7ed, ASSIGN 2024-12-03T23:14:40,230 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4e6706440af7674db97bb48a05858115, server=5f9d64419733,41167,1733267673270 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,230 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=67, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 256869bf1dc8aa33be79adb19e13d134, server=5f9d64419733,41167,1733267673270:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 256869bf1dc8aa33be79adb19e13d134, server=5f9d64419733,41167,1733267673270 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,230 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 23a123891f32f583441cd7569a6de1f2, server=5f9d64419733,44899,1733267673202 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,231 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2b7da3736c6b257b43fe66ea13807bc6, server=5f9d64419733,41167,1733267673270 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,231 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6d9981bd2d541debb970a37ac121e7ed, server=5f9d64419733,44899,1733267673202 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,231 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 553e1b6e96399dd43d3b8f7cfad54903, server=5f9d64419733,41167,1733267673270 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,231 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 12 msec 2024-12-03T23:14:40,232 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=61 2024-12-03T23:14:40,232 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=5f9d64419733,41167,1733267673270, table=hbase:meta, region=1588230740 2024-12-03T23:14:40,232 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 5f9d64419733,41167,1733267673270, state=OPEN 2024-12-03T23:14:40,233 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44117 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T23:14:40,235 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41167-0x1017a9bb4a60003, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T23:14:40,235 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44899-0x1017a9bb4a60002, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T23:14:40,235 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44117-0x1017a9bb4a60000, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T23:14:40,235 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T23:14:40,235 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T23:14:40,236 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T23:14:40,236 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=23a123891f32f583441cd7569a6de1f2, ASSIGN to state=OFFLINE, location=null, table=test, region=23a123891f32f583441cd7569a6de1f2 to restore RIT 2024-12-03T23:14:40,236 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b7da3736c6b257b43fe66ea13807bc6, ASSIGN to state=OFFLINE, location=null, table=test, region=2b7da3736c6b257b43fe66ea13807bc6 to restore RIT 2024-12-03T23:14:40,237 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e6706440af7674db97bb48a05858115, ASSIGN to state=OFFLINE, location=null, table=test, region=4e6706440af7674db97bb48a05858115 to restore RIT 2024-12-03T23:14:40,237 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=256869bf1dc8aa33be79adb19e13d134, ASSIGN to state=OFFLINE, location=null, table=test, region=256869bf1dc8aa33be79adb19e13d134 to restore RIT 2024-12-03T23:14:40,237 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=742c43a49ea3b763b8232c8e8c8966b3, ASSIGN to state=OFFLINE, location=null, table=test, region=742c43a49ea3b763b8232c8e8c8966b3 to restore RIT 2024-12-03T23:14:40,237 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=553e1b6e96399dd43d3b8f7cfad54903, ASSIGN to state=OFFLINE, location=null, table=test, region=553e1b6e96399dd43d3b8f7cfad54903 to restore RIT 2024-12-03T23:14:40,237 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=384b2e2791a8bf699cb7996e1aa7469c, ASSIGN to state=OFFLINE, location=null, table=test, region=384b2e2791a8bf699cb7996e1aa7469c to restore RIT 2024-12-03T23:14:40,237 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6d9981bd2d541debb970a37ac121e7ed, ASSIGN to state=OFFLINE, location=null, table=test, region=6d9981bd2d541debb970a37ac121e7ed to restore RIT 2024-12-03T23:14:40,237 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733267710237 2024-12-03T23:14:40,237 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-03T23:14:40,238 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 256869bf1dc8aa33be79adb19e13d134, server=5f9d64419733,41167,1733267673270:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5f9d64419733,46833,1733267673070, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T23:14:40,239 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-03T23:14:40,240 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-03T23:14:40,241 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=76, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6d9981bd2d541debb970a37ac121e7ed, server=5f9d64419733,44899,1733267673202 2024-12-03T23:14:40,241 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T23:14:40,241 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=76, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6d9981bd2d541debb970a37ac121e7ed, server=5f9d64419733,44899,1733267673202 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T23:14:40,246 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d9981bd2d541debb970a37ac121e7ed, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,246 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f71f90ef2d2f59e885f2ad6ca1a99596, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,246 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e35e57b6f6cbff01d3e782fbf6efd1f, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,246 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=55eff67791415c655cef877182b9bbe1, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,246 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=212a9963c9cddede26a4eb521cff5254, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,246 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f514b27fbd5c6ef451e11852713ea906, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,247 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=73cd6e72f89a0d14d1d979f76c9c07a4, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,247 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=256869bf1dc8aa33be79adb19e13d134, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,247 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2db7cef12ba09a51b0a7204fc81deeec, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,247 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=384b2e2791a8bf699cb7996e1aa7469c, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,247 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5ddac1aa9a6f0af338b83490d6d952c2, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,247 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1d026a7a0318e7deb79fa16b1262ae2c, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,248 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4e6706440af7674db97bb48a05858115, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,248 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f00ca5d778884039f3248393a629dcfc, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,248 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5e6925efcdb64a8de08b4c4bbf3a3d2e, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,248 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=742c43a49ea3b763b8232c8e8c8966b3, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,248 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=10beaf4047510d958e4a559194242a57, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,248 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=89a698454f10c9fb6cfc108c35d8415a, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,249 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b7da3736c6b257b43fe66ea13807bc6, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,249 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a6de975926600fad463e13238b2ac8a9, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,249 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e5e4ecc042f874ec5567f7e86548256f, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,249 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23a123891f32f583441cd7569a6de1f2, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,249 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0f09ea7cbe3ceec5232d11c21a512a4, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,249 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e96b12de28d6c3047803f54a3b027459, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,249 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=553e1b6e96399dd43d3b8f7cfad54903, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,250 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=db9924063978f5acb21b073c5d94a7ad, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,250 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-03T23:14:40,250 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733267740250 2024-12-03T23:14:40,250 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733267800250 2024-12-03T23:14:40,250 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 12 msec 2024-12-03T23:14:40,250 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-03T23:14:40,250 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T23:14:40,250 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-03T23:14:40,250 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-03T23:14:40,250 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-03T23:14:40,250 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-03T23:14:40,250 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-03T23:14:40,251 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-03T23:14:40,251 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-03T23:14:40,254 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-03T23:14:40,254 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T23:14:40,254 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=64, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-03T23:14:40,254 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T23:14:40,254 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-03T23:14:40,254 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T23:14:40,254 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-03T23:14:40,254 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T23:14:40,255 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-03T23:14:40,255 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T23:14:40,255 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T23:14:40,255 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T23:14:40,255 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T23:14:40,255 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-03T23:14:40,255 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-03T23:14:40,255 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-03T23:14:40,255 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T23:14:40,255 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-03T23:14:40,255 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-03T23:14:40,255 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-03T23:14:40,255 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-03T23:14:40,256 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-03T23:14:40,258 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 256869bf1dc8aa33be79adb19e13d134, server=5f9d64419733,41167,1733267673270:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5f9d64419733,46833,1733267673070, splitWal=true, meta=true 2024-12-03T23:14:40,258 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5f9d64419733%2C46833%2C1733267673070.meta.1733267675470.meta, worker=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,258 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5f9d64419733%2C46833%2C1733267673070.meta.1733267675470.meta, worker=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,258 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-03T23:14:40,258 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,258 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5f9d64419733%2C46833%2C1733267673070.1733267675041, worker=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,258 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5f9d64419733%2C46833%2C1733267673070.1733267675041, worker=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,258 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=23a123891f32f583441cd7569a6de1f2, ASSIGN 2024-12-03T23:14:40,258 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b7da3736c6b257b43fe66ea13807bc6, ASSIGN 2024-12-03T23:14:40,258 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e6706440af7674db97bb48a05858115, ASSIGN 2024-12-03T23:14:40,258 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=256869bf1dc8aa33be79adb19e13d134, ASSIGN 2024-12-03T23:14:40,258 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=742c43a49ea3b763b8232c8e8c8966b3, ASSIGN 2024-12-03T23:14:40,258 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=553e1b6e96399dd43d3b8f7cfad54903, ASSIGN 2024-12-03T23:14:40,258 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=384b2e2791a8bf699cb7996e1aa7469c, ASSIGN 2024-12-03T23:14:40,258 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6d9981bd2d541debb970a37ac121e7ed, ASSIGN 2024-12-03T23:14:40,258 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4e6706440af7674db97bb48a05858115, server=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,258 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=67, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 256869bf1dc8aa33be79adb19e13d134, server=5f9d64419733,41167,1733267673270:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 256869bf1dc8aa33be79adb19e13d134, server=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,258 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 23a123891f32f583441cd7569a6de1f2, server=5f9d64419733,44899,1733267673202 2024-12-03T23:14:40,258 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2b7da3736c6b257b43fe66ea13807bc6, server=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,258 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6d9981bd2d541debb970a37ac121e7ed, server=5f9d64419733,44899,1733267673202 2024-12-03T23:14:40,258 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 256869bf1dc8aa33be79adb19e13d134, server=5f9d64419733,41167,1733267673270:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5f9d64419733,46833,1733267673070, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,258 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5f9d64419733%2C46833%2C1733267673070.meta.1733267675470.meta, worker=5f9d64419733,41167,1733267673270 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,258 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5f9d64419733%2C46833%2C1733267673070.meta.1733267675470.meta, worker=5f9d64419733,41167,1733267673270 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,258 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,258 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=5f9d64419733,41167,1733267673270 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,258 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5f9d64419733%2C46833%2C1733267673070.1733267675041, worker=5f9d64419733,41167,1733267673270 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,258 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5f9d64419733%2C46833%2C1733267673070.1733267675041, worker=5f9d64419733,41167,1733267673270 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,259 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=23a123891f32f583441cd7569a6de1f2, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,259 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=23a123891f32f583441cd7569a6de1f2, ASSIGN 2024-12-03T23:14:40,259 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b7da3736c6b257b43fe66ea13807bc6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,259 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b7da3736c6b257b43fe66ea13807bc6, ASSIGN 2024-12-03T23:14:40,259 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e6706440af7674db97bb48a05858115, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,259 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e6706440af7674db97bb48a05858115, ASSIGN 2024-12-03T23:14:40,259 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=256869bf1dc8aa33be79adb19e13d134, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,259 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=256869bf1dc8aa33be79adb19e13d134, ASSIGN 2024-12-03T23:14:40,259 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=742c43a49ea3b763b8232c8e8c8966b3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,259 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=742c43a49ea3b763b8232c8e8c8966b3, ASSIGN 2024-12-03T23:14:40,259 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=553e1b6e96399dd43d3b8f7cfad54903, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,259 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=553e1b6e96399dd43d3b8f7cfad54903, ASSIGN 2024-12-03T23:14:40,259 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=384b2e2791a8bf699cb7996e1aa7469c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,259 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=384b2e2791a8bf699cb7996e1aa7469c, ASSIGN 2024-12-03T23:14:40,259 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6d9981bd2d541debb970a37ac121e7ed, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,259 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6d9981bd2d541debb970a37ac121e7ed, ASSIGN 2024-12-03T23:14:40,259 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4e6706440af7674db97bb48a05858115, server=5f9d64419733,41167,1733267673270 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,259 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=67, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 256869bf1dc8aa33be79adb19e13d134, server=5f9d64419733,41167,1733267673270:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 256869bf1dc8aa33be79adb19e13d134, server=5f9d64419733,41167,1733267673270 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,259 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 23a123891f32f583441cd7569a6de1f2, server=5f9d64419733,44899,1733267673202 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,259 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2b7da3736c6b257b43fe66ea13807bc6, server=5f9d64419733,41167,1733267673270 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,259 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6d9981bd2d541debb970a37ac121e7ed, server=5f9d64419733,44899,1733267673202 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,259 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-12-03T23:14:40,260 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=61 2024-12-03T23:14:40,260 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=5f9d64419733,41167,1733267673270, table=hbase:meta, region=1588230740 2024-12-03T23:14:40,260 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 5f9d64419733,41167,1733267673270, state=OPEN 2024-12-03T23:14:40,262 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44899-0x1017a9bb4a60002, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T23:14:40,262 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41167-0x1017a9bb4a60003, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T23:14:40,262 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44117-0x1017a9bb4a60000, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T23:14:40,262 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T23:14:40,262 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T23:14:40,262 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T23:14:40,263 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=23a123891f32f583441cd7569a6de1f2, ASSIGN to state=OFFLINE, location=null, table=test, region=23a123891f32f583441cd7569a6de1f2 to restore RIT 2024-12-03T23:14:40,263 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b7da3736c6b257b43fe66ea13807bc6, ASSIGN to state=OFFLINE, location=null, table=test, region=2b7da3736c6b257b43fe66ea13807bc6 to restore RIT 2024-12-03T23:14:40,263 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e6706440af7674db97bb48a05858115, ASSIGN to state=OFFLINE, location=null, table=test, region=4e6706440af7674db97bb48a05858115 to restore RIT 2024-12-03T23:14:40,263 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=256869bf1dc8aa33be79adb19e13d134, ASSIGN to state=OFFLINE, location=null, table=test, region=256869bf1dc8aa33be79adb19e13d134 to restore RIT 2024-12-03T23:14:40,263 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=742c43a49ea3b763b8232c8e8c8966b3, ASSIGN to state=OFFLINE, location=null, table=test, region=742c43a49ea3b763b8232c8e8c8966b3 to restore RIT 2024-12-03T23:14:40,263 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=553e1b6e96399dd43d3b8f7cfad54903, ASSIGN to state=OFFLINE, location=null, table=test, region=553e1b6e96399dd43d3b8f7cfad54903 to restore RIT 2024-12-03T23:14:40,263 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=384b2e2791a8bf699cb7996e1aa7469c, ASSIGN to state=OFFLINE, location=null, table=test, region=384b2e2791a8bf699cb7996e1aa7469c to restore RIT 2024-12-03T23:14:40,264 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6d9981bd2d541debb970a37ac121e7ed, ASSIGN to state=OFFLINE, location=null, table=test, region=6d9981bd2d541debb970a37ac121e7ed to restore RIT 2024-12-03T23:14:40,264 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733267710264 2024-12-03T23:14:40,264 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-03T23:14:40,264 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 256869bf1dc8aa33be79adb19e13d134, server=5f9d64419733,41167,1733267673270:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5f9d64419733,46833,1733267673070, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T23:14:40,266 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-03T23:14:40,267 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-03T23:14:40,267 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=75, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2b7da3736c6b257b43fe66ea13807bc6, server=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,267 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T23:14:40,267 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=75, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2b7da3736c6b257b43fe66ea13807bc6, server=5f9d64419733,41167,1733267673270 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T23:14:40,272 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d9981bd2d541debb970a37ac121e7ed, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,272 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f71f90ef2d2f59e885f2ad6ca1a99596, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,272 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e35e57b6f6cbff01d3e782fbf6efd1f, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,272 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=55eff67791415c655cef877182b9bbe1, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,273 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=212a9963c9cddede26a4eb521cff5254, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,273 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f514b27fbd5c6ef451e11852713ea906, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,273 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=73cd6e72f89a0d14d1d979f76c9c07a4, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,273 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=256869bf1dc8aa33be79adb19e13d134, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,273 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2db7cef12ba09a51b0a7204fc81deeec, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,273 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=384b2e2791a8bf699cb7996e1aa7469c, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,273 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5ddac1aa9a6f0af338b83490d6d952c2, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,274 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1d026a7a0318e7deb79fa16b1262ae2c, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,274 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4e6706440af7674db97bb48a05858115, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,274 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f00ca5d778884039f3248393a629dcfc, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,274 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5e6925efcdb64a8de08b4c4bbf3a3d2e, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,274 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=742c43a49ea3b763b8232c8e8c8966b3, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,274 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=10beaf4047510d958e4a559194242a57, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,275 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=89a698454f10c9fb6cfc108c35d8415a, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,275 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b7da3736c6b257b43fe66ea13807bc6, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,275 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a6de975926600fad463e13238b2ac8a9, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,275 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e5e4ecc042f874ec5567f7e86548256f, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,275 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23a123891f32f583441cd7569a6de1f2, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,275 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0f09ea7cbe3ceec5232d11c21a512a4, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,276 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e96b12de28d6c3047803f54a3b027459, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,276 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=553e1b6e96399dd43d3b8f7cfad54903, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,276 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=db9924063978f5acb21b073c5d94a7ad, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,276 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-03T23:14:40,276 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733267740276 2024-12-03T23:14:40,276 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733267800276 2024-12-03T23:14:40,276 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 11 msec 2024-12-03T23:14:40,276 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-03T23:14:40,276 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T23:14:40,276 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-03T23:14:40,276 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-03T23:14:40,276 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-03T23:14:40,276 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-03T23:14:40,276 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-03T23:14:40,277 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-03T23:14:40,277 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-03T23:14:40,280 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-03T23:14:40,280 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T23:14:40,280 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=64, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-03T23:14:40,280 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T23:14:40,280 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-03T23:14:40,280 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T23:14:40,280 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-03T23:14:40,280 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T23:14:40,281 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-03T23:14:40,281 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T23:14:40,281 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T23:14:40,281 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T23:14:40,281 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T23:14:40,281 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-03T23:14:40,281 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-03T23:14:40,281 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T23:14:40,281 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-03T23:14:40,281 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-03T23:14:40,281 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-03T23:14:40,281 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-03T23:14:40,282 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-03T23:14:40,283 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 256869bf1dc8aa33be79adb19e13d134, server=5f9d64419733,41167,1733267673270:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5f9d64419733,46833,1733267673070, splitWal=true, meta=true 2024-12-03T23:14:40,283 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5f9d64419733%2C46833%2C1733267673070.meta.1733267675470.meta, worker=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,283 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5f9d64419733%2C46833%2C1733267673070.meta.1733267675470.meta, worker=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,283 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-03T23:14:40,283 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,283 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5f9d64419733%2C46833%2C1733267673070.1733267675041, worker=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,283 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5f9d64419733%2C46833%2C1733267673070.1733267675041, worker=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,283 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=23a123891f32f583441cd7569a6de1f2, ASSIGN 2024-12-03T23:14:40,283 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b7da3736c6b257b43fe66ea13807bc6, ASSIGN 2024-12-03T23:14:40,283 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e6706440af7674db97bb48a05858115, ASSIGN 2024-12-03T23:14:40,283 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=256869bf1dc8aa33be79adb19e13d134, ASSIGN 2024-12-03T23:14:40,283 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=742c43a49ea3b763b8232c8e8c8966b3, ASSIGN 2024-12-03T23:14:40,284 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=553e1b6e96399dd43d3b8f7cfad54903, ASSIGN 2024-12-03T23:14:40,284 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=384b2e2791a8bf699cb7996e1aa7469c, ASSIGN 2024-12-03T23:14:40,284 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6d9981bd2d541debb970a37ac121e7ed, ASSIGN 2024-12-03T23:14:40,284 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4e6706440af7674db97bb48a05858115, server=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,284 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=67, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 256869bf1dc8aa33be79adb19e13d134, server=5f9d64419733,41167,1733267673270:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 256869bf1dc8aa33be79adb19e13d134, server=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,284 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 23a123891f32f583441cd7569a6de1f2, server=5f9d64419733,44899,1733267673202 2024-12-03T23:14:40,284 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2b7da3736c6b257b43fe66ea13807bc6, server=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,284 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 256869bf1dc8aa33be79adb19e13d134, server=5f9d64419733,41167,1733267673270:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5f9d64419733,46833,1733267673070, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,284 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5f9d64419733%2C46833%2C1733267673070.meta.1733267675470.meta, worker=5f9d64419733,41167,1733267673270 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,284 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5f9d64419733%2C46833%2C1733267673070.meta.1733267675470.meta, worker=5f9d64419733,41167,1733267673270 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,284 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,284 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=5f9d64419733,41167,1733267673270 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,284 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5f9d64419733%2C46833%2C1733267673070.1733267675041, worker=5f9d64419733,41167,1733267673270 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,284 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5f9d64419733%2C46833%2C1733267673070.1733267675041, worker=5f9d64419733,41167,1733267673270 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,284 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=23a123891f32f583441cd7569a6de1f2, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,284 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=23a123891f32f583441cd7569a6de1f2, ASSIGN 2024-12-03T23:14:40,284 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b7da3736c6b257b43fe66ea13807bc6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,284 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b7da3736c6b257b43fe66ea13807bc6, ASSIGN 2024-12-03T23:14:40,284 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e6706440af7674db97bb48a05858115, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,284 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e6706440af7674db97bb48a05858115, ASSIGN 2024-12-03T23:14:40,284 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=256869bf1dc8aa33be79adb19e13d134, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,284 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=256869bf1dc8aa33be79adb19e13d134, ASSIGN 2024-12-03T23:14:40,284 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=742c43a49ea3b763b8232c8e8c8966b3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,284 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=742c43a49ea3b763b8232c8e8c8966b3, ASSIGN 2024-12-03T23:14:40,284 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=553e1b6e96399dd43d3b8f7cfad54903, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,284 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=553e1b6e96399dd43d3b8f7cfad54903, ASSIGN 2024-12-03T23:14:40,284 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=384b2e2791a8bf699cb7996e1aa7469c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,284 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=384b2e2791a8bf699cb7996e1aa7469c, ASSIGN 2024-12-03T23:14:40,284 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6d9981bd2d541debb970a37ac121e7ed, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,284 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6d9981bd2d541debb970a37ac121e7ed, ASSIGN 2024-12-03T23:14:40,284 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4e6706440af7674db97bb48a05858115, server=5f9d64419733,41167,1733267673270 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,284 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=67, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 256869bf1dc8aa33be79adb19e13d134, server=5f9d64419733,41167,1733267673270:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 256869bf1dc8aa33be79adb19e13d134, server=5f9d64419733,41167,1733267673270 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,284 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 23a123891f32f583441cd7569a6de1f2, server=5f9d64419733,44899,1733267673202 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,284 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2b7da3736c6b257b43fe66ea13807bc6, server=5f9d64419733,41167,1733267673270 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,284 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-12-03T23:14:40,285 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=61 2024-12-03T23:14:40,285 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=5f9d64419733,41167,1733267673270, table=hbase:meta, region=1588230740 2024-12-03T23:14:40,285 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 5f9d64419733,41167,1733267673270, state=OPEN 2024-12-03T23:14:40,287 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41167-0x1017a9bb4a60003, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T23:14:40,287 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44899-0x1017a9bb4a60002, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T23:14:40,287 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44117-0x1017a9bb4a60000, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T23:14:40,287 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T23:14:40,287 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T23:14:40,287 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T23:14:40,289 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=23a123891f32f583441cd7569a6de1f2, ASSIGN to state=OFFLINE, location=null, table=test, region=23a123891f32f583441cd7569a6de1f2 to restore RIT 2024-12-03T23:14:40,289 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b7da3736c6b257b43fe66ea13807bc6, ASSIGN to state=OFFLINE, location=null, table=test, region=2b7da3736c6b257b43fe66ea13807bc6 to restore RIT 2024-12-03T23:14:40,289 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e6706440af7674db97bb48a05858115, ASSIGN to state=OFFLINE, location=null, table=test, region=4e6706440af7674db97bb48a05858115 to restore RIT 2024-12-03T23:14:40,289 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=256869bf1dc8aa33be79adb19e13d134, ASSIGN to state=OFFLINE, location=null, table=test, region=256869bf1dc8aa33be79adb19e13d134 to restore RIT 2024-12-03T23:14:40,289 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=742c43a49ea3b763b8232c8e8c8966b3, ASSIGN to state=OFFLINE, location=null, table=test, region=742c43a49ea3b763b8232c8e8c8966b3 to restore RIT 2024-12-03T23:14:40,289 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=553e1b6e96399dd43d3b8f7cfad54903, ASSIGN to state=OFFLINE, location=null, table=test, region=553e1b6e96399dd43d3b8f7cfad54903 to restore RIT 2024-12-03T23:14:40,289 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=384b2e2791a8bf699cb7996e1aa7469c, ASSIGN to state=OFFLINE, location=null, table=test, region=384b2e2791a8bf699cb7996e1aa7469c to restore RIT 2024-12-03T23:14:40,289 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6d9981bd2d541debb970a37ac121e7ed, ASSIGN to state=OFFLINE, location=null, table=test, region=6d9981bd2d541debb970a37ac121e7ed to restore RIT 2024-12-03T23:14:40,290 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733267710290 2024-12-03T23:14:40,290 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-03T23:14:40,290 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 256869bf1dc8aa33be79adb19e13d134, server=5f9d64419733,41167,1733267673270:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5f9d64419733,46833,1733267673070, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T23:14:40,291 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-03T23:14:40,293 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-03T23:14:40,293 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=74, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 23a123891f32f583441cd7569a6de1f2, server=5f9d64419733,44899,1733267673202 2024-12-03T23:14:40,293 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T23:14:40,293 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=74, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 23a123891f32f583441cd7569a6de1f2, server=5f9d64419733,44899,1733267673202 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T23:14:40,297 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d9981bd2d541debb970a37ac121e7ed, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,298 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f71f90ef2d2f59e885f2ad6ca1a99596, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,298 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e35e57b6f6cbff01d3e782fbf6efd1f, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,298 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=55eff67791415c655cef877182b9bbe1, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,298 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=212a9963c9cddede26a4eb521cff5254, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,298 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f514b27fbd5c6ef451e11852713ea906, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,298 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=73cd6e72f89a0d14d1d979f76c9c07a4, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,299 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=256869bf1dc8aa33be79adb19e13d134, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,299 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2db7cef12ba09a51b0a7204fc81deeec, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,299 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=384b2e2791a8bf699cb7996e1aa7469c, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,299 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5ddac1aa9a6f0af338b83490d6d952c2, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,299 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1d026a7a0318e7deb79fa16b1262ae2c, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,299 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4e6706440af7674db97bb48a05858115, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,299 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f00ca5d778884039f3248393a629dcfc, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,299 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5e6925efcdb64a8de08b4c4bbf3a3d2e, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,300 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=742c43a49ea3b763b8232c8e8c8966b3, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,300 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=10beaf4047510d958e4a559194242a57, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,300 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=89a698454f10c9fb6cfc108c35d8415a, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,300 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b7da3736c6b257b43fe66ea13807bc6, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,300 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a6de975926600fad463e13238b2ac8a9, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,300 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e5e4ecc042f874ec5567f7e86548256f, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,300 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23a123891f32f583441cd7569a6de1f2, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,300 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0f09ea7cbe3ceec5232d11c21a512a4, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,301 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e96b12de28d6c3047803f54a3b027459, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,301 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=553e1b6e96399dd43d3b8f7cfad54903, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,301 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=db9924063978f5acb21b073c5d94a7ad, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,301 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-03T23:14:40,301 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733267740301 2024-12-03T23:14:40,301 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733267800301 2024-12-03T23:14:40,301 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 11 msec 2024-12-03T23:14:40,301 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-03T23:14:40,301 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T23:14:40,301 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-03T23:14:40,301 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-03T23:14:40,301 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-03T23:14:40,301 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-03T23:14:40,302 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-03T23:14:40,302 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-03T23:14:40,302 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-03T23:14:40,304 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-03T23:14:40,304 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T23:14:40,304 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=64, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-03T23:14:40,304 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T23:14:40,305 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T23:14:40,305 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-03T23:14:40,305 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T23:14:40,305 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-03T23:14:40,305 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T23:14:40,305 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T23:14:40,305 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T23:14:40,305 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T23:14:40,305 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-03T23:14:40,305 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-03T23:14:40,305 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T23:14:40,305 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-03T23:14:40,305 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-03T23:14:40,305 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-03T23:14:40,305 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-03T23:14:40,306 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-03T23:14:40,307 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 256869bf1dc8aa33be79adb19e13d134, server=5f9d64419733,41167,1733267673270:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5f9d64419733,46833,1733267673070, splitWal=true, meta=true 2024-12-03T23:14:40,307 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5f9d64419733%2C46833%2C1733267673070.meta.1733267675470.meta, worker=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,307 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5f9d64419733%2C46833%2C1733267673070.meta.1733267675470.meta, worker=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,307 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-03T23:14:40,307 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,307 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5f9d64419733%2C46833%2C1733267673070.1733267675041, worker=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,307 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5f9d64419733%2C46833%2C1733267673070.1733267675041, worker=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,307 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=23a123891f32f583441cd7569a6de1f2, ASSIGN 2024-12-03T23:14:40,307 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b7da3736c6b257b43fe66ea13807bc6, ASSIGN 2024-12-03T23:14:40,307 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e6706440af7674db97bb48a05858115, ASSIGN 2024-12-03T23:14:40,307 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=256869bf1dc8aa33be79adb19e13d134, ASSIGN 2024-12-03T23:14:40,307 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=742c43a49ea3b763b8232c8e8c8966b3, ASSIGN 2024-12-03T23:14:40,307 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=553e1b6e96399dd43d3b8f7cfad54903, ASSIGN 2024-12-03T23:14:40,307 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=384b2e2791a8bf699cb7996e1aa7469c, ASSIGN 2024-12-03T23:14:40,307 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6d9981bd2d541debb970a37ac121e7ed, ASSIGN 2024-12-03T23:14:40,307 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4e6706440af7674db97bb48a05858115, server=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,308 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=67, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 256869bf1dc8aa33be79adb19e13d134, server=5f9d64419733,41167,1733267673270:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 256869bf1dc8aa33be79adb19e13d134, server=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,308 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 23a123891f32f583441cd7569a6de1f2, server=5f9d64419733,44899,1733267673202 2024-12-03T23:14:40,308 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 256869bf1dc8aa33be79adb19e13d134, server=5f9d64419733,41167,1733267673270:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5f9d64419733,46833,1733267673070, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,308 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5f9d64419733%2C46833%2C1733267673070.meta.1733267675470.meta, worker=5f9d64419733,41167,1733267673270 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,308 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5f9d64419733%2C46833%2C1733267673070.meta.1733267675470.meta, worker=5f9d64419733,41167,1733267673270 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,308 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,308 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=5f9d64419733,41167,1733267673270 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,308 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5f9d64419733%2C46833%2C1733267673070.1733267675041, worker=5f9d64419733,41167,1733267673270 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,308 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5f9d64419733%2C46833%2C1733267673070.1733267675041, worker=5f9d64419733,41167,1733267673270 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,308 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=23a123891f32f583441cd7569a6de1f2, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,308 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=23a123891f32f583441cd7569a6de1f2, ASSIGN 2024-12-03T23:14:40,308 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b7da3736c6b257b43fe66ea13807bc6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,308 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b7da3736c6b257b43fe66ea13807bc6, ASSIGN 2024-12-03T23:14:40,308 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e6706440af7674db97bb48a05858115, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,308 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e6706440af7674db97bb48a05858115, ASSIGN 2024-12-03T23:14:40,308 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=256869bf1dc8aa33be79adb19e13d134, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,308 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=256869bf1dc8aa33be79adb19e13d134, ASSIGN 2024-12-03T23:14:40,308 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=742c43a49ea3b763b8232c8e8c8966b3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,308 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=742c43a49ea3b763b8232c8e8c8966b3, ASSIGN 2024-12-03T23:14:40,308 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=553e1b6e96399dd43d3b8f7cfad54903, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,308 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=553e1b6e96399dd43d3b8f7cfad54903, ASSIGN 2024-12-03T23:14:40,308 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=384b2e2791a8bf699cb7996e1aa7469c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,308 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=384b2e2791a8bf699cb7996e1aa7469c, ASSIGN 2024-12-03T23:14:40,308 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6d9981bd2d541debb970a37ac121e7ed, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,308 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6d9981bd2d541debb970a37ac121e7ed, ASSIGN 2024-12-03T23:14:40,308 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4e6706440af7674db97bb48a05858115, server=5f9d64419733,41167,1733267673270 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,308 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=67, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 256869bf1dc8aa33be79adb19e13d134, server=5f9d64419733,41167,1733267673270:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 256869bf1dc8aa33be79adb19e13d134, server=5f9d64419733,41167,1733267673270 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,308 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 23a123891f32f583441cd7569a6de1f2, server=5f9d64419733,44899,1733267673202 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,308 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-12-03T23:14:40,309 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=61 2024-12-03T23:14:40,309 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=5f9d64419733,41167,1733267673270, table=hbase:meta, region=1588230740 2024-12-03T23:14:40,309 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 5f9d64419733,41167,1733267673270, state=OPEN 2024-12-03T23:14:40,311 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41167-0x1017a9bb4a60003, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T23:14:40,311 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44899-0x1017a9bb4a60002, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T23:14:40,311 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44117-0x1017a9bb4a60000, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T23:14:40,311 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T23:14:40,311 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T23:14:40,311 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T23:14:40,312 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=23a123891f32f583441cd7569a6de1f2, ASSIGN to state=OFFLINE, location=null, table=test, region=23a123891f32f583441cd7569a6de1f2 to restore RIT 2024-12-03T23:14:40,312 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b7da3736c6b257b43fe66ea13807bc6, ASSIGN to state=OFFLINE, location=null, table=test, region=2b7da3736c6b257b43fe66ea13807bc6 to restore RIT 2024-12-03T23:14:40,312 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e6706440af7674db97bb48a05858115, ASSIGN to state=OFFLINE, location=null, table=test, region=4e6706440af7674db97bb48a05858115 to restore RIT 2024-12-03T23:14:40,312 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=256869bf1dc8aa33be79adb19e13d134, ASSIGN to state=OFFLINE, location=null, table=test, region=256869bf1dc8aa33be79adb19e13d134 to restore RIT 2024-12-03T23:14:40,313 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=742c43a49ea3b763b8232c8e8c8966b3, ASSIGN to state=OFFLINE, location=null, table=test, region=742c43a49ea3b763b8232c8e8c8966b3 to restore RIT 2024-12-03T23:14:40,313 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=553e1b6e96399dd43d3b8f7cfad54903, ASSIGN to state=OFFLINE, location=null, table=test, region=553e1b6e96399dd43d3b8f7cfad54903 to restore RIT 2024-12-03T23:14:40,313 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=384b2e2791a8bf699cb7996e1aa7469c, ASSIGN to state=OFFLINE, location=null, table=test, region=384b2e2791a8bf699cb7996e1aa7469c to restore RIT 2024-12-03T23:14:40,313 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6d9981bd2d541debb970a37ac121e7ed, ASSIGN to state=OFFLINE, location=null, table=test, region=6d9981bd2d541debb970a37ac121e7ed to restore RIT 2024-12-03T23:14:40,313 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733267710313 2024-12-03T23:14:40,313 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-03T23:14:40,313 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 256869bf1dc8aa33be79adb19e13d134, server=5f9d64419733,41167,1733267673270:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5f9d64419733,46833,1733267673070, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T23:14:40,314 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-03T23:14:40,316 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-03T23:14:40,316 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=73, ppid=67, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 256869bf1dc8aa33be79adb19e13d134, server=5f9d64419733,41167,1733267673270:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 256869bf1dc8aa33be79adb19e13d134, server=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,316 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T23:14:40,316 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=73, ppid=67, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 256869bf1dc8aa33be79adb19e13d134, server=5f9d64419733,41167,1733267673270:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 256869bf1dc8aa33be79adb19e13d134, server=5f9d64419733,41167,1733267673270 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T23:14:40,320 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d9981bd2d541debb970a37ac121e7ed, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,320 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f71f90ef2d2f59e885f2ad6ca1a99596, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,321 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e35e57b6f6cbff01d3e782fbf6efd1f, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,321 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=55eff67791415c655cef877182b9bbe1, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,321 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=212a9963c9cddede26a4eb521cff5254, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,321 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f514b27fbd5c6ef451e11852713ea906, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,321 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=73cd6e72f89a0d14d1d979f76c9c07a4, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,322 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=256869bf1dc8aa33be79adb19e13d134, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,322 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2db7cef12ba09a51b0a7204fc81deeec, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,322 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=384b2e2791a8bf699cb7996e1aa7469c, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,322 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5ddac1aa9a6f0af338b83490d6d952c2, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,322 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1d026a7a0318e7deb79fa16b1262ae2c, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,323 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4e6706440af7674db97bb48a05858115, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,323 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f00ca5d778884039f3248393a629dcfc, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,323 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5e6925efcdb64a8de08b4c4bbf3a3d2e, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,323 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=742c43a49ea3b763b8232c8e8c8966b3, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,323 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=10beaf4047510d958e4a559194242a57, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,324 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=89a698454f10c9fb6cfc108c35d8415a, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,324 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b7da3736c6b257b43fe66ea13807bc6, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,324 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a6de975926600fad463e13238b2ac8a9, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,324 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e5e4ecc042f874ec5567f7e86548256f, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,324 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23a123891f32f583441cd7569a6de1f2, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,325 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0f09ea7cbe3ceec5232d11c21a512a4, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,325 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e96b12de28d6c3047803f54a3b027459, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,325 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=553e1b6e96399dd43d3b8f7cfad54903, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,325 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=db9924063978f5acb21b073c5d94a7ad, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,325 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-03T23:14:40,325 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733267740325 2024-12-03T23:14:40,325 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733267800325 2024-12-03T23:14:40,326 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 12 msec 2024-12-03T23:14:40,326 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-03T23:14:40,326 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T23:14:40,326 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-03T23:14:40,326 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-03T23:14:40,326 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-03T23:14:40,326 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-03T23:14:40,326 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-03T23:14:40,327 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-03T23:14:40,327 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-03T23:14:40,329 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-03T23:14:40,329 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T23:14:40,329 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T23:14:40,330 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T23:14:40,330 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-03T23:14:40,330 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T23:14:40,330 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-03T23:14:40,330 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T23:14:40,330 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T23:14:40,330 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T23:14:40,330 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T23:14:40,330 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-03T23:14:40,330 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-03T23:14:40,330 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T23:14:40,330 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-03T23:14:40,330 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-03T23:14:40,330 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-03T23:14:40,330 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-03T23:14:40,331 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-03T23:14:40,331 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44117 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T23:14:40,333 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 256869bf1dc8aa33be79adb19e13d134, server=5f9d64419733,41167,1733267673270:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5f9d64419733,46833,1733267673070, splitWal=true, meta=true 2024-12-03T23:14:40,333 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5f9d64419733%2C46833%2C1733267673070.meta.1733267675470.meta, worker=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,333 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5f9d64419733%2C46833%2C1733267673070.meta.1733267675470.meta, worker=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,333 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-03T23:14:40,333 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,333 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5f9d64419733%2C46833%2C1733267673070.1733267675041, worker=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,333 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5f9d64419733%2C46833%2C1733267673070.1733267675041, worker=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,333 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=23a123891f32f583441cd7569a6de1f2, ASSIGN 2024-12-03T23:14:40,333 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b7da3736c6b257b43fe66ea13807bc6, ASSIGN 2024-12-03T23:14:40,333 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e6706440af7674db97bb48a05858115, ASSIGN 2024-12-03T23:14:40,333 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=256869bf1dc8aa33be79adb19e13d134, ASSIGN 2024-12-03T23:14:40,333 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=742c43a49ea3b763b8232c8e8c8966b3, ASSIGN 2024-12-03T23:14:40,333 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=553e1b6e96399dd43d3b8f7cfad54903, ASSIGN 2024-12-03T23:14:40,333 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=384b2e2791a8bf699cb7996e1aa7469c, ASSIGN 2024-12-03T23:14:40,333 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6d9981bd2d541debb970a37ac121e7ed, ASSIGN 2024-12-03T23:14:40,333 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4e6706440af7674db97bb48a05858115, server=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,333 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=67, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 256869bf1dc8aa33be79adb19e13d134, server=5f9d64419733,41167,1733267673270:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 256869bf1dc8aa33be79adb19e13d134, server=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,333 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 256869bf1dc8aa33be79adb19e13d134, server=5f9d64419733,41167,1733267673270:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5f9d64419733,46833,1733267673070, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,333 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5f9d64419733%2C46833%2C1733267673070.meta.1733267675470.meta, worker=5f9d64419733,41167,1733267673270 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,333 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5f9d64419733%2C46833%2C1733267673070.meta.1733267675470.meta, worker=5f9d64419733,41167,1733267673270 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,333 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,333 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=5f9d64419733,41167,1733267673270 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,333 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5f9d64419733%2C46833%2C1733267673070.1733267675041, worker=5f9d64419733,41167,1733267673270 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,333 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5f9d64419733%2C46833%2C1733267673070.1733267675041, worker=5f9d64419733,41167,1733267673270 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,333 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=23a123891f32f583441cd7569a6de1f2, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,333 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=23a123891f32f583441cd7569a6de1f2, ASSIGN 2024-12-03T23:14:40,333 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b7da3736c6b257b43fe66ea13807bc6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,333 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b7da3736c6b257b43fe66ea13807bc6, ASSIGN 2024-12-03T23:14:40,333 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e6706440af7674db97bb48a05858115, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,333 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e6706440af7674db97bb48a05858115, ASSIGN 2024-12-03T23:14:40,333 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=256869bf1dc8aa33be79adb19e13d134, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,334 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=256869bf1dc8aa33be79adb19e13d134, ASSIGN 2024-12-03T23:14:40,334 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=742c43a49ea3b763b8232c8e8c8966b3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,334 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=742c43a49ea3b763b8232c8e8c8966b3, ASSIGN 2024-12-03T23:14:40,334 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=553e1b6e96399dd43d3b8f7cfad54903, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,334 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=553e1b6e96399dd43d3b8f7cfad54903, ASSIGN 2024-12-03T23:14:40,334 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=384b2e2791a8bf699cb7996e1aa7469c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,334 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=384b2e2791a8bf699cb7996e1aa7469c, ASSIGN 2024-12-03T23:14:40,334 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6d9981bd2d541debb970a37ac121e7ed, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,334 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6d9981bd2d541debb970a37ac121e7ed, ASSIGN 2024-12-03T23:14:40,334 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4e6706440af7674db97bb48a05858115, server=5f9d64419733,41167,1733267673270 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,334 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=67, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 256869bf1dc8aa33be79adb19e13d134, server=5f9d64419733,41167,1733267673270:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 256869bf1dc8aa33be79adb19e13d134, server=5f9d64419733,41167,1733267673270 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,334 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-12-03T23:14:40,335 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=61 2024-12-03T23:14:40,335 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=5f9d64419733,41167,1733267673270, table=hbase:meta, region=1588230740 2024-12-03T23:14:40,335 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 5f9d64419733,41167,1733267673270, state=OPEN 2024-12-03T23:14:40,337 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44117 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T23:14:40,337 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41167-0x1017a9bb4a60003, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T23:14:40,337 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44117-0x1017a9bb4a60000, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T23:14:40,337 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T23:14:40,337 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T23:14:40,337 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44899-0x1017a9bb4a60002, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T23:14:40,338 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T23:14:40,338 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=23a123891f32f583441cd7569a6de1f2, ASSIGN to state=OFFLINE, location=null, table=test, region=23a123891f32f583441cd7569a6de1f2 to restore RIT 2024-12-03T23:14:40,338 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b7da3736c6b257b43fe66ea13807bc6, ASSIGN to state=OFFLINE, location=null, table=test, region=2b7da3736c6b257b43fe66ea13807bc6 to restore RIT 2024-12-03T23:14:40,338 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e6706440af7674db97bb48a05858115, ASSIGN to state=OFFLINE, location=null, table=test, region=4e6706440af7674db97bb48a05858115 to restore RIT 2024-12-03T23:14:40,338 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=256869bf1dc8aa33be79adb19e13d134, ASSIGN to state=OFFLINE, location=null, table=test, region=256869bf1dc8aa33be79adb19e13d134 to restore RIT 2024-12-03T23:14:40,339 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=742c43a49ea3b763b8232c8e8c8966b3, ASSIGN to state=OFFLINE, location=null, table=test, region=742c43a49ea3b763b8232c8e8c8966b3 to restore RIT 2024-12-03T23:14:40,339 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=553e1b6e96399dd43d3b8f7cfad54903, ASSIGN to state=OFFLINE, location=null, table=test, region=553e1b6e96399dd43d3b8f7cfad54903 to restore RIT 2024-12-03T23:14:40,339 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=384b2e2791a8bf699cb7996e1aa7469c, ASSIGN to state=OFFLINE, location=null, table=test, region=384b2e2791a8bf699cb7996e1aa7469c to restore RIT 2024-12-03T23:14:40,339 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6d9981bd2d541debb970a37ac121e7ed, ASSIGN to state=OFFLINE, location=null, table=test, region=6d9981bd2d541debb970a37ac121e7ed to restore RIT 2024-12-03T23:14:40,339 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733267710339 2024-12-03T23:14:40,339 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-03T23:14:40,339 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 256869bf1dc8aa33be79adb19e13d134, server=5f9d64419733,41167,1733267673270:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5f9d64419733,46833,1733267673070, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T23:14:40,340 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-03T23:14:40,342 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-03T23:14:40,342 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=72, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4e6706440af7674db97bb48a05858115, server=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,342 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T23:14:40,342 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=72, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4e6706440af7674db97bb48a05858115, server=5f9d64419733,41167,1733267673270 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T23:14:40,345 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d9981bd2d541debb970a37ac121e7ed, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,345 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f71f90ef2d2f59e885f2ad6ca1a99596, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,345 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e35e57b6f6cbff01d3e782fbf6efd1f, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,345 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=55eff67791415c655cef877182b9bbe1, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,346 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=212a9963c9cddede26a4eb521cff5254, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,346 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f514b27fbd5c6ef451e11852713ea906, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,346 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=73cd6e72f89a0d14d1d979f76c9c07a4, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,346 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=256869bf1dc8aa33be79adb19e13d134, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,346 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2db7cef12ba09a51b0a7204fc81deeec, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,346 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=384b2e2791a8bf699cb7996e1aa7469c, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,346 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5ddac1aa9a6f0af338b83490d6d952c2, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,346 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1d026a7a0318e7deb79fa16b1262ae2c, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,347 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4e6706440af7674db97bb48a05858115, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,347 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f00ca5d778884039f3248393a629dcfc, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,347 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5e6925efcdb64a8de08b4c4bbf3a3d2e, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,347 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=742c43a49ea3b763b8232c8e8c8966b3, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,347 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=10beaf4047510d958e4a559194242a57, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,347 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=89a698454f10c9fb6cfc108c35d8415a, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,347 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b7da3736c6b257b43fe66ea13807bc6, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,347 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a6de975926600fad463e13238b2ac8a9, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,347 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e5e4ecc042f874ec5567f7e86548256f, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,348 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23a123891f32f583441cd7569a6de1f2, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,348 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0f09ea7cbe3ceec5232d11c21a512a4, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,348 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e96b12de28d6c3047803f54a3b027459, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,348 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=553e1b6e96399dd43d3b8f7cfad54903, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,348 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=db9924063978f5acb21b073c5d94a7ad, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,348 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-03T23:14:40,348 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733267740348 2024-12-03T23:14:40,348 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733267800348 2024-12-03T23:14:40,348 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-12-03T23:14:40,348 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-03T23:14:40,348 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T23:14:40,348 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-03T23:14:40,348 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-03T23:14:40,349 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-03T23:14:40,349 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-03T23:14:40,349 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-03T23:14:40,349 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-03T23:14:40,350 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-03T23:14:40,351 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-03T23:14:40,351 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T23:14:40,352 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T23:14:40,352 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T23:14:40,352 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-03T23:14:40,352 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T23:14:40,352 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T23:14:40,352 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T23:14:40,352 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T23:14:40,352 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T23:14:40,352 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-03T23:14:40,352 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-03T23:14:40,352 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T23:14:40,352 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-03T23:14:40,352 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-03T23:14:40,352 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-03T23:14:40,352 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-03T23:14:40,353 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-03T23:14:40,355 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 256869bf1dc8aa33be79adb19e13d134, server=5f9d64419733,41167,1733267673270:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5f9d64419733,46833,1733267673070, splitWal=true, meta=true 2024-12-03T23:14:40,355 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5f9d64419733%2C46833%2C1733267673070.meta.1733267675470.meta, worker=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,355 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5f9d64419733%2C46833%2C1733267673070.meta.1733267675470.meta, worker=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,355 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-03T23:14:40,355 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,355 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5f9d64419733%2C46833%2C1733267673070.1733267675041, worker=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,355 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5f9d64419733%2C46833%2C1733267673070.1733267675041, worker=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,355 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=23a123891f32f583441cd7569a6de1f2, ASSIGN 2024-12-03T23:14:40,355 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b7da3736c6b257b43fe66ea13807bc6, ASSIGN 2024-12-03T23:14:40,355 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e6706440af7674db97bb48a05858115, ASSIGN 2024-12-03T23:14:40,355 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=256869bf1dc8aa33be79adb19e13d134, ASSIGN 2024-12-03T23:14:40,355 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=742c43a49ea3b763b8232c8e8c8966b3, ASSIGN 2024-12-03T23:14:40,355 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=553e1b6e96399dd43d3b8f7cfad54903, ASSIGN 2024-12-03T23:14:40,355 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=384b2e2791a8bf699cb7996e1aa7469c, ASSIGN 2024-12-03T23:14:40,355 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6d9981bd2d541debb970a37ac121e7ed, ASSIGN 2024-12-03T23:14:40,355 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4e6706440af7674db97bb48a05858115, server=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,355 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 256869bf1dc8aa33be79adb19e13d134, server=5f9d64419733,41167,1733267673270:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5f9d64419733,46833,1733267673070, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,355 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5f9d64419733%2C46833%2C1733267673070.meta.1733267675470.meta, worker=5f9d64419733,41167,1733267673270 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,355 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5f9d64419733%2C46833%2C1733267673070.meta.1733267675470.meta, worker=5f9d64419733,41167,1733267673270 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,355 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,355 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=5f9d64419733,41167,1733267673270 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,355 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5f9d64419733%2C46833%2C1733267673070.1733267675041, worker=5f9d64419733,41167,1733267673270 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,355 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5f9d64419733%2C46833%2C1733267673070.1733267675041, worker=5f9d64419733,41167,1733267673270 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,355 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=23a123891f32f583441cd7569a6de1f2, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,355 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=23a123891f32f583441cd7569a6de1f2, ASSIGN 2024-12-03T23:14:40,355 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b7da3736c6b257b43fe66ea13807bc6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,355 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b7da3736c6b257b43fe66ea13807bc6, ASSIGN 2024-12-03T23:14:40,355 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e6706440af7674db97bb48a05858115, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,355 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e6706440af7674db97bb48a05858115, ASSIGN 2024-12-03T23:14:40,356 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=256869bf1dc8aa33be79adb19e13d134, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,356 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=256869bf1dc8aa33be79adb19e13d134, ASSIGN 2024-12-03T23:14:40,356 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=742c43a49ea3b763b8232c8e8c8966b3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,356 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=742c43a49ea3b763b8232c8e8c8966b3, ASSIGN 2024-12-03T23:14:40,356 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=553e1b6e96399dd43d3b8f7cfad54903, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,356 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=553e1b6e96399dd43d3b8f7cfad54903, ASSIGN 2024-12-03T23:14:40,356 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=384b2e2791a8bf699cb7996e1aa7469c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,356 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=384b2e2791a8bf699cb7996e1aa7469c, ASSIGN 2024-12-03T23:14:40,356 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6d9981bd2d541debb970a37ac121e7ed, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,356 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6d9981bd2d541debb970a37ac121e7ed, ASSIGN 2024-12-03T23:14:40,356 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4e6706440af7674db97bb48a05858115, server=5f9d64419733,41167,1733267673270 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,356 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-12-03T23:14:40,357 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=61 2024-12-03T23:14:40,357 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=5f9d64419733,41167,1733267673270, table=hbase:meta, region=1588230740 2024-12-03T23:14:40,357 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 5f9d64419733,41167,1733267673270, state=OPEN 2024-12-03T23:14:40,359 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44117-0x1017a9bb4a60000, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T23:14:40,359 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41167-0x1017a9bb4a60003, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T23:14:40,359 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44899-0x1017a9bb4a60002, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T23:14:40,359 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T23:14:40,359 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T23:14:40,359 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T23:14:40,360 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=23a123891f32f583441cd7569a6de1f2, ASSIGN to state=OFFLINE, location=null, table=test, region=23a123891f32f583441cd7569a6de1f2 to restore RIT 2024-12-03T23:14:40,360 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b7da3736c6b257b43fe66ea13807bc6, ASSIGN to state=OFFLINE, location=null, table=test, region=2b7da3736c6b257b43fe66ea13807bc6 to restore RIT 2024-12-03T23:14:40,360 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e6706440af7674db97bb48a05858115, ASSIGN to state=OFFLINE, location=null, table=test, region=4e6706440af7674db97bb48a05858115 to restore RIT 2024-12-03T23:14:40,360 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=256869bf1dc8aa33be79adb19e13d134, ASSIGN to state=OFFLINE, location=null, table=test, region=256869bf1dc8aa33be79adb19e13d134 to restore RIT 2024-12-03T23:14:40,360 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=742c43a49ea3b763b8232c8e8c8966b3, ASSIGN to state=OFFLINE, location=null, table=test, region=742c43a49ea3b763b8232c8e8c8966b3 to restore RIT 2024-12-03T23:14:40,360 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=553e1b6e96399dd43d3b8f7cfad54903, ASSIGN to state=OFFLINE, location=null, table=test, region=553e1b6e96399dd43d3b8f7cfad54903 to restore RIT 2024-12-03T23:14:40,360 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=384b2e2791a8bf699cb7996e1aa7469c, ASSIGN to state=OFFLINE, location=null, table=test, region=384b2e2791a8bf699cb7996e1aa7469c to restore RIT 2024-12-03T23:14:40,360 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6d9981bd2d541debb970a37ac121e7ed, ASSIGN to state=OFFLINE, location=null, table=test, region=6d9981bd2d541debb970a37ac121e7ed to restore RIT 2024-12-03T23:14:40,360 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733267710360 2024-12-03T23:14:40,360 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-03T23:14:40,360 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 256869bf1dc8aa33be79adb19e13d134, server=5f9d64419733,41167,1733267673270:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5f9d64419733,46833,1733267673070, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T23:14:40,362 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-03T23:14:40,363 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-03T23:14:40,363 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6d9981bd2d541debb970a37ac121e7ed, ASSIGN 2024-12-03T23:14:40,363 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T23:14:40,363 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6d9981bd2d541debb970a37ac121e7ed, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T23:14:40,365 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d9981bd2d541debb970a37ac121e7ed, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,365 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f71f90ef2d2f59e885f2ad6ca1a99596, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,365 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e35e57b6f6cbff01d3e782fbf6efd1f, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,365 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=55eff67791415c655cef877182b9bbe1, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,366 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=212a9963c9cddede26a4eb521cff5254, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,366 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f514b27fbd5c6ef451e11852713ea906, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,366 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=73cd6e72f89a0d14d1d979f76c9c07a4, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,366 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=256869bf1dc8aa33be79adb19e13d134, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,366 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2db7cef12ba09a51b0a7204fc81deeec, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,366 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=384b2e2791a8bf699cb7996e1aa7469c, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,366 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5ddac1aa9a6f0af338b83490d6d952c2, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,366 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1d026a7a0318e7deb79fa16b1262ae2c, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,366 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4e6706440af7674db97bb48a05858115, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,366 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f00ca5d778884039f3248393a629dcfc, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,367 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5e6925efcdb64a8de08b4c4bbf3a3d2e, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,367 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=742c43a49ea3b763b8232c8e8c8966b3, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,367 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=10beaf4047510d958e4a559194242a57, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,367 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=89a698454f10c9fb6cfc108c35d8415a, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,367 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b7da3736c6b257b43fe66ea13807bc6, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,367 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a6de975926600fad463e13238b2ac8a9, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,367 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e5e4ecc042f874ec5567f7e86548256f, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,367 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23a123891f32f583441cd7569a6de1f2, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,367 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0f09ea7cbe3ceec5232d11c21a512a4, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,367 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e96b12de28d6c3047803f54a3b027459, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,368 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=553e1b6e96399dd43d3b8f7cfad54903, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,368 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=db9924063978f5acb21b073c5d94a7ad, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,368 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-03T23:14:40,368 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733267740368 2024-12-03T23:14:40,368 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733267800368 2024-12-03T23:14:40,368 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-12-03T23:14:40,368 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-03T23:14:40,368 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T23:14:40,368 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-03T23:14:40,368 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-03T23:14:40,368 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-03T23:14:40,368 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-03T23:14:40,368 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-03T23:14:40,368 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-03T23:14:40,369 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-03T23:14:40,370 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-03T23:14:40,371 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T23:14:40,371 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T23:14:40,371 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T23:14:40,371 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T23:14:40,371 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T23:14:40,371 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T23:14:40,371 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T23:14:40,371 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T23:14:40,371 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-03T23:14:40,371 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-03T23:14:40,371 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T23:14:40,371 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-03T23:14:40,371 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-03T23:14:40,371 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-03T23:14:40,371 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-03T23:14:40,372 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-03T23:14:40,373 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 256869bf1dc8aa33be79adb19e13d134, server=5f9d64419733,41167,1733267673270:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5f9d64419733,46833,1733267673070, splitWal=true, meta=true 2024-12-03T23:14:40,373 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5f9d64419733%2C46833%2C1733267673070.meta.1733267675470.meta, worker=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,373 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5f9d64419733%2C46833%2C1733267673070.meta.1733267675470.meta, worker=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,373 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-03T23:14:40,373 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,373 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5f9d64419733%2C46833%2C1733267673070.1733267675041, worker=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,373 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5f9d64419733%2C46833%2C1733267673070.1733267675041, worker=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,373 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=23a123891f32f583441cd7569a6de1f2, ASSIGN 2024-12-03T23:14:40,373 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b7da3736c6b257b43fe66ea13807bc6, ASSIGN 2024-12-03T23:14:40,373 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e6706440af7674db97bb48a05858115, ASSIGN 2024-12-03T23:14:40,373 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=256869bf1dc8aa33be79adb19e13d134, ASSIGN 2024-12-03T23:14:40,373 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=742c43a49ea3b763b8232c8e8c8966b3, ASSIGN 2024-12-03T23:14:40,373 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=553e1b6e96399dd43d3b8f7cfad54903, ASSIGN 2024-12-03T23:14:40,373 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=384b2e2791a8bf699cb7996e1aa7469c, ASSIGN 2024-12-03T23:14:40,373 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6d9981bd2d541debb970a37ac121e7ed, ASSIGN 2024-12-03T23:14:40,373 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 256869bf1dc8aa33be79adb19e13d134, server=5f9d64419733,41167,1733267673270:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5f9d64419733,46833,1733267673070, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,373 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5f9d64419733%2C46833%2C1733267673070.meta.1733267675470.meta, worker=5f9d64419733,41167,1733267673270 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,373 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5f9d64419733%2C46833%2C1733267673070.meta.1733267675470.meta, worker=5f9d64419733,41167,1733267673270 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,373 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,373 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=5f9d64419733,41167,1733267673270 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,373 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5f9d64419733%2C46833%2C1733267673070.1733267675041, worker=5f9d64419733,41167,1733267673270 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,373 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5f9d64419733%2C46833%2C1733267673070.1733267675041, worker=5f9d64419733,41167,1733267673270 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,373 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=23a123891f32f583441cd7569a6de1f2, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,374 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=23a123891f32f583441cd7569a6de1f2, ASSIGN 2024-12-03T23:14:40,374 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b7da3736c6b257b43fe66ea13807bc6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,374 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b7da3736c6b257b43fe66ea13807bc6, ASSIGN 2024-12-03T23:14:40,374 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e6706440af7674db97bb48a05858115, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,374 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e6706440af7674db97bb48a05858115, ASSIGN 2024-12-03T23:14:40,374 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=256869bf1dc8aa33be79adb19e13d134, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,374 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=256869bf1dc8aa33be79adb19e13d134, ASSIGN 2024-12-03T23:14:40,374 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=742c43a49ea3b763b8232c8e8c8966b3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,374 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=742c43a49ea3b763b8232c8e8c8966b3, ASSIGN 2024-12-03T23:14:40,374 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=553e1b6e96399dd43d3b8f7cfad54903, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,374 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=553e1b6e96399dd43d3b8f7cfad54903, ASSIGN 2024-12-03T23:14:40,374 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=384b2e2791a8bf699cb7996e1aa7469c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,374 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=384b2e2791a8bf699cb7996e1aa7469c, ASSIGN 2024-12-03T23:14:40,374 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6d9981bd2d541debb970a37ac121e7ed, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,374 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6d9981bd2d541debb970a37ac121e7ed, ASSIGN 2024-12-03T23:14:40,374 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-12-03T23:14:40,375 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=61 2024-12-03T23:14:40,375 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=5f9d64419733,41167,1733267673270, table=hbase:meta, region=1588230740 2024-12-03T23:14:40,375 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 5f9d64419733,41167,1733267673270, state=OPEN 2024-12-03T23:14:40,377 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44117-0x1017a9bb4a60000, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T23:14:40,377 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44899-0x1017a9bb4a60002, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T23:14:40,377 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41167-0x1017a9bb4a60003, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T23:14:40,377 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T23:14:40,377 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T23:14:40,377 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T23:14:40,377 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=23a123891f32f583441cd7569a6de1f2, ASSIGN to state=OFFLINE, location=null, table=test, region=23a123891f32f583441cd7569a6de1f2 to restore RIT 2024-12-03T23:14:40,377 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b7da3736c6b257b43fe66ea13807bc6, ASSIGN to state=OFFLINE, location=null, table=test, region=2b7da3736c6b257b43fe66ea13807bc6 to restore RIT 2024-12-03T23:14:40,378 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e6706440af7674db97bb48a05858115, ASSIGN to state=OFFLINE, location=null, table=test, region=4e6706440af7674db97bb48a05858115 to restore RIT 2024-12-03T23:14:40,378 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=256869bf1dc8aa33be79adb19e13d134, ASSIGN to state=OFFLINE, location=null, table=test, region=256869bf1dc8aa33be79adb19e13d134 to restore RIT 2024-12-03T23:14:40,378 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=742c43a49ea3b763b8232c8e8c8966b3, ASSIGN to state=OFFLINE, location=null, table=test, region=742c43a49ea3b763b8232c8e8c8966b3 to restore RIT 2024-12-03T23:14:40,378 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=553e1b6e96399dd43d3b8f7cfad54903, ASSIGN to state=OFFLINE, location=null, table=test, region=553e1b6e96399dd43d3b8f7cfad54903 to restore RIT 2024-12-03T23:14:40,378 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=384b2e2791a8bf699cb7996e1aa7469c, ASSIGN to state=OFFLINE, location=null, table=test, region=384b2e2791a8bf699cb7996e1aa7469c to restore RIT 2024-12-03T23:14:40,378 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6d9981bd2d541debb970a37ac121e7ed, ASSIGN to state=OFFLINE, location=null, table=test, region=6d9981bd2d541debb970a37ac121e7ed to restore RIT 2024-12-03T23:14:40,380 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733267710380 2024-12-03T23:14:40,380 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-03T23:14:40,380 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 256869bf1dc8aa33be79adb19e13d134, server=5f9d64419733,41167,1733267673270:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5f9d64419733,46833,1733267673070, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T23:14:40,381 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-03T23:14:40,382 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-03T23:14:40,382 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=384b2e2791a8bf699cb7996e1aa7469c, ASSIGN 2024-12-03T23:14:40,382 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T23:14:40,382 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=384b2e2791a8bf699cb7996e1aa7469c, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T23:14:40,384 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d9981bd2d541debb970a37ac121e7ed, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,384 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f71f90ef2d2f59e885f2ad6ca1a99596, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,385 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e35e57b6f6cbff01d3e782fbf6efd1f, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,385 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=55eff67791415c655cef877182b9bbe1, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,385 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=212a9963c9cddede26a4eb521cff5254, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,385 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f514b27fbd5c6ef451e11852713ea906, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,385 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=73cd6e72f89a0d14d1d979f76c9c07a4, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,385 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=256869bf1dc8aa33be79adb19e13d134, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,385 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2db7cef12ba09a51b0a7204fc81deeec, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,385 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=384b2e2791a8bf699cb7996e1aa7469c, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,385 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5ddac1aa9a6f0af338b83490d6d952c2, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,386 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1d026a7a0318e7deb79fa16b1262ae2c, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,386 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4e6706440af7674db97bb48a05858115, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,386 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f00ca5d778884039f3248393a629dcfc, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,386 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5e6925efcdb64a8de08b4c4bbf3a3d2e, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,386 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=742c43a49ea3b763b8232c8e8c8966b3, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,386 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=10beaf4047510d958e4a559194242a57, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,386 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=89a698454f10c9fb6cfc108c35d8415a, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,386 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b7da3736c6b257b43fe66ea13807bc6, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,386 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a6de975926600fad463e13238b2ac8a9, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,387 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e5e4ecc042f874ec5567f7e86548256f, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,387 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23a123891f32f583441cd7569a6de1f2, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,387 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0f09ea7cbe3ceec5232d11c21a512a4, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,387 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e96b12de28d6c3047803f54a3b027459, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,387 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=553e1b6e96399dd43d3b8f7cfad54903, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,387 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=db9924063978f5acb21b073c5d94a7ad, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,387 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-03T23:14:40,387 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733267740387 2024-12-03T23:14:40,387 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733267800387 2024-12-03T23:14:40,387 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 6 msec 2024-12-03T23:14:40,387 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-03T23:14:40,387 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T23:14:40,387 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-03T23:14:40,387 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-03T23:14:40,388 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-03T23:14:40,388 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-03T23:14:40,388 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-03T23:14:40,388 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-03T23:14:40,388 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-03T23:14:40,390 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-03T23:14:40,390 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T23:14:40,390 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T23:14:40,390 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T23:14:40,390 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T23:14:40,390 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T23:14:40,390 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T23:14:40,390 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T23:14:40,390 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-03T23:14:40,390 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-03T23:14:40,390 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T23:14:40,390 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-03T23:14:40,390 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-03T23:14:40,390 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-03T23:14:40,390 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-03T23:14:40,391 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-03T23:14:40,391 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 256869bf1dc8aa33be79adb19e13d134, server=5f9d64419733,41167,1733267673270:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5f9d64419733,46833,1733267673070, splitWal=true, meta=true 2024-12-03T23:14:40,392 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5f9d64419733%2C46833%2C1733267673070.meta.1733267675470.meta, worker=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,392 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5f9d64419733%2C46833%2C1733267673070.meta.1733267675470.meta, worker=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,392 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-03T23:14:40,392 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,392 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5f9d64419733%2C46833%2C1733267673070.1733267675041, worker=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,392 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5f9d64419733%2C46833%2C1733267673070.1733267675041, worker=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,392 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=23a123891f32f583441cd7569a6de1f2, ASSIGN 2024-12-03T23:14:40,392 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b7da3736c6b257b43fe66ea13807bc6, ASSIGN 2024-12-03T23:14:40,392 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e6706440af7674db97bb48a05858115, ASSIGN 2024-12-03T23:14:40,392 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=256869bf1dc8aa33be79adb19e13d134, ASSIGN 2024-12-03T23:14:40,392 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=742c43a49ea3b763b8232c8e8c8966b3, ASSIGN 2024-12-03T23:14:40,392 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=553e1b6e96399dd43d3b8f7cfad54903, ASSIGN 2024-12-03T23:14:40,392 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=384b2e2791a8bf699cb7996e1aa7469c, ASSIGN 2024-12-03T23:14:40,392 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 256869bf1dc8aa33be79adb19e13d134, server=5f9d64419733,41167,1733267673270:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5f9d64419733,46833,1733267673070, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,392 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5f9d64419733%2C46833%2C1733267673070.meta.1733267675470.meta, worker=5f9d64419733,41167,1733267673270 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,392 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5f9d64419733%2C46833%2C1733267673070.meta.1733267675470.meta, worker=5f9d64419733,41167,1733267673270 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,392 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,392 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=5f9d64419733,41167,1733267673270 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,392 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5f9d64419733%2C46833%2C1733267673070.1733267675041, worker=5f9d64419733,41167,1733267673270 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,392 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5f9d64419733%2C46833%2C1733267673070.1733267675041, worker=5f9d64419733,41167,1733267673270 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,392 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=23a123891f32f583441cd7569a6de1f2, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,392 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=23a123891f32f583441cd7569a6de1f2, ASSIGN 2024-12-03T23:14:40,392 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b7da3736c6b257b43fe66ea13807bc6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,392 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b7da3736c6b257b43fe66ea13807bc6, ASSIGN 2024-12-03T23:14:40,392 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e6706440af7674db97bb48a05858115, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,392 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e6706440af7674db97bb48a05858115, ASSIGN 2024-12-03T23:14:40,392 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=256869bf1dc8aa33be79adb19e13d134, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,392 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=256869bf1dc8aa33be79adb19e13d134, ASSIGN 2024-12-03T23:14:40,392 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=742c43a49ea3b763b8232c8e8c8966b3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,392 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=742c43a49ea3b763b8232c8e8c8966b3, ASSIGN 2024-12-03T23:14:40,393 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=553e1b6e96399dd43d3b8f7cfad54903, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,393 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=553e1b6e96399dd43d3b8f7cfad54903, ASSIGN 2024-12-03T23:14:40,393 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=384b2e2791a8bf699cb7996e1aa7469c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,393 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=384b2e2791a8bf699cb7996e1aa7469c, ASSIGN 2024-12-03T23:14:40,393 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-12-03T23:14:40,393 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=61 2024-12-03T23:14:40,393 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=5f9d64419733,41167,1733267673270, table=hbase:meta, region=1588230740 2024-12-03T23:14:40,394 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 5f9d64419733,41167,1733267673270, state=OPEN 2024-12-03T23:14:40,395 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44899-0x1017a9bb4a60002, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T23:14:40,395 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44117-0x1017a9bb4a60000, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T23:14:40,395 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41167-0x1017a9bb4a60003, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T23:14:40,395 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T23:14:40,395 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T23:14:40,395 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T23:14:40,396 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=23a123891f32f583441cd7569a6de1f2, ASSIGN to state=OFFLINE, location=null, table=test, region=23a123891f32f583441cd7569a6de1f2 to restore RIT 2024-12-03T23:14:40,396 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b7da3736c6b257b43fe66ea13807bc6, ASSIGN to state=OFFLINE, location=null, table=test, region=2b7da3736c6b257b43fe66ea13807bc6 to restore RIT 2024-12-03T23:14:40,396 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e6706440af7674db97bb48a05858115, ASSIGN to state=OFFLINE, location=null, table=test, region=4e6706440af7674db97bb48a05858115 to restore RIT 2024-12-03T23:14:40,396 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=256869bf1dc8aa33be79adb19e13d134, ASSIGN to state=OFFLINE, location=null, table=test, region=256869bf1dc8aa33be79adb19e13d134 to restore RIT 2024-12-03T23:14:40,396 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=742c43a49ea3b763b8232c8e8c8966b3, ASSIGN to state=OFFLINE, location=null, table=test, region=742c43a49ea3b763b8232c8e8c8966b3 to restore RIT 2024-12-03T23:14:40,396 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=553e1b6e96399dd43d3b8f7cfad54903, ASSIGN to state=OFFLINE, location=null, table=test, region=553e1b6e96399dd43d3b8f7cfad54903 to restore RIT 2024-12-03T23:14:40,396 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=384b2e2791a8bf699cb7996e1aa7469c, ASSIGN to state=OFFLINE, location=null, table=test, region=384b2e2791a8bf699cb7996e1aa7469c to restore RIT 2024-12-03T23:14:40,397 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733267710397 2024-12-03T23:14:40,397 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-03T23:14:40,397 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 256869bf1dc8aa33be79adb19e13d134, server=5f9d64419733,41167,1733267673270:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5f9d64419733,46833,1733267673070, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T23:14:40,398 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-03T23:14:40,400 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-03T23:14:40,400 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=553e1b6e96399dd43d3b8f7cfad54903, ASSIGN 2024-12-03T23:14:40,400 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T23:14:40,400 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=553e1b6e96399dd43d3b8f7cfad54903, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T23:14:40,401 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d9981bd2d541debb970a37ac121e7ed, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,401 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f71f90ef2d2f59e885f2ad6ca1a99596, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,401 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e35e57b6f6cbff01d3e782fbf6efd1f, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,401 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=55eff67791415c655cef877182b9bbe1, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,401 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=212a9963c9cddede26a4eb521cff5254, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,401 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f514b27fbd5c6ef451e11852713ea906, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,401 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=73cd6e72f89a0d14d1d979f76c9c07a4, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,401 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=256869bf1dc8aa33be79adb19e13d134, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,401 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2db7cef12ba09a51b0a7204fc81deeec, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,401 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=384b2e2791a8bf699cb7996e1aa7469c, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,402 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5ddac1aa9a6f0af338b83490d6d952c2, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,402 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1d026a7a0318e7deb79fa16b1262ae2c, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,402 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4e6706440af7674db97bb48a05858115, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,402 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f00ca5d778884039f3248393a629dcfc, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,402 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5e6925efcdb64a8de08b4c4bbf3a3d2e, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,402 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=742c43a49ea3b763b8232c8e8c8966b3, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,402 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=10beaf4047510d958e4a559194242a57, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,402 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=89a698454f10c9fb6cfc108c35d8415a, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,402 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b7da3736c6b257b43fe66ea13807bc6, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,402 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a6de975926600fad463e13238b2ac8a9, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,402 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e5e4ecc042f874ec5567f7e86548256f, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,403 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23a123891f32f583441cd7569a6de1f2, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,403 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0f09ea7cbe3ceec5232d11c21a512a4, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,403 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e96b12de28d6c3047803f54a3b027459, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,403 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=553e1b6e96399dd43d3b8f7cfad54903, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,403 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=db9924063978f5acb21b073c5d94a7ad, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,403 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-03T23:14:40,403 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733267740403 2024-12-03T23:14:40,403 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733267800403 2024-12-03T23:14:40,403 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 6 msec 2024-12-03T23:14:40,403 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-03T23:14:40,403 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T23:14:40,403 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-03T23:14:40,403 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-03T23:14:40,403 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-03T23:14:40,403 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-03T23:14:40,403 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-03T23:14:40,404 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-03T23:14:40,404 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-03T23:14:40,405 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-03T23:14:40,405 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T23:14:40,405 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T23:14:40,406 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T23:14:40,406 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T23:14:40,406 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T23:14:40,406 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T23:14:40,406 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-03T23:14:40,406 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-03T23:14:40,406 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T23:14:40,406 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-03T23:14:40,406 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-03T23:14:40,406 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-03T23:14:40,406 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-03T23:14:40,406 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-03T23:14:40,407 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 256869bf1dc8aa33be79adb19e13d134, server=5f9d64419733,41167,1733267673270:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5f9d64419733,46833,1733267673070, splitWal=true, meta=true 2024-12-03T23:14:40,407 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5f9d64419733%2C46833%2C1733267673070.meta.1733267675470.meta, worker=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,407 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5f9d64419733%2C46833%2C1733267673070.meta.1733267675470.meta, worker=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,407 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-03T23:14:40,407 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,407 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5f9d64419733%2C46833%2C1733267673070.1733267675041, worker=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,407 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5f9d64419733%2C46833%2C1733267673070.1733267675041, worker=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,407 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=23a123891f32f583441cd7569a6de1f2, ASSIGN 2024-12-03T23:14:40,407 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b7da3736c6b257b43fe66ea13807bc6, ASSIGN 2024-12-03T23:14:40,407 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e6706440af7674db97bb48a05858115, ASSIGN 2024-12-03T23:14:40,407 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=256869bf1dc8aa33be79adb19e13d134, ASSIGN 2024-12-03T23:14:40,407 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=742c43a49ea3b763b8232c8e8c8966b3, ASSIGN 2024-12-03T23:14:40,407 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=553e1b6e96399dd43d3b8f7cfad54903, ASSIGN 2024-12-03T23:14:40,407 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 256869bf1dc8aa33be79adb19e13d134, server=5f9d64419733,41167,1733267673270:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5f9d64419733,46833,1733267673070, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,407 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5f9d64419733%2C46833%2C1733267673070.meta.1733267675470.meta, worker=5f9d64419733,41167,1733267673270 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,408 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5f9d64419733%2C46833%2C1733267673070.meta.1733267675470.meta, worker=5f9d64419733,41167,1733267673270 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,408 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,408 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=5f9d64419733,41167,1733267673270 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,408 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5f9d64419733%2C46833%2C1733267673070.1733267675041, worker=5f9d64419733,41167,1733267673270 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,408 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5f9d64419733%2C46833%2C1733267673070.1733267675041, worker=5f9d64419733,41167,1733267673270 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,408 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=23a123891f32f583441cd7569a6de1f2, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,408 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=23a123891f32f583441cd7569a6de1f2, ASSIGN 2024-12-03T23:14:40,408 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b7da3736c6b257b43fe66ea13807bc6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,408 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b7da3736c6b257b43fe66ea13807bc6, ASSIGN 2024-12-03T23:14:40,408 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e6706440af7674db97bb48a05858115, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,408 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e6706440af7674db97bb48a05858115, ASSIGN 2024-12-03T23:14:40,408 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=256869bf1dc8aa33be79adb19e13d134, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,408 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=256869bf1dc8aa33be79adb19e13d134, ASSIGN 2024-12-03T23:14:40,408 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=742c43a49ea3b763b8232c8e8c8966b3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,408 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=742c43a49ea3b763b8232c8e8c8966b3, ASSIGN 2024-12-03T23:14:40,408 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=553e1b6e96399dd43d3b8f7cfad54903, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,408 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=553e1b6e96399dd43d3b8f7cfad54903, ASSIGN 2024-12-03T23:14:40,408 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-12-03T23:14:40,408 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=61 2024-12-03T23:14:40,409 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=5f9d64419733,41167,1733267673270, table=hbase:meta, region=1588230740 2024-12-03T23:14:40,409 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 5f9d64419733,41167,1733267673270, state=OPEN 2024-12-03T23:14:40,412 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41167-0x1017a9bb4a60003, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T23:14:40,412 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T23:14:40,412 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44899-0x1017a9bb4a60002, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T23:14:40,412 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44117-0x1017a9bb4a60000, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T23:14:40,412 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T23:14:40,412 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T23:14:40,413 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=23a123891f32f583441cd7569a6de1f2, ASSIGN to state=OFFLINE, location=null, table=test, region=23a123891f32f583441cd7569a6de1f2 to restore RIT 2024-12-03T23:14:40,413 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b7da3736c6b257b43fe66ea13807bc6, ASSIGN to state=OFFLINE, location=null, table=test, region=2b7da3736c6b257b43fe66ea13807bc6 to restore RIT 2024-12-03T23:14:40,413 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e6706440af7674db97bb48a05858115, ASSIGN to state=OFFLINE, location=null, table=test, region=4e6706440af7674db97bb48a05858115 to restore RIT 2024-12-03T23:14:40,413 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=256869bf1dc8aa33be79adb19e13d134, ASSIGN to state=OFFLINE, location=null, table=test, region=256869bf1dc8aa33be79adb19e13d134 to restore RIT 2024-12-03T23:14:40,413 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=742c43a49ea3b763b8232c8e8c8966b3, ASSIGN to state=OFFLINE, location=null, table=test, region=742c43a49ea3b763b8232c8e8c8966b3 to restore RIT 2024-12-03T23:14:40,413 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=553e1b6e96399dd43d3b8f7cfad54903, ASSIGN to state=OFFLINE, location=null, table=test, region=553e1b6e96399dd43d3b8f7cfad54903 to restore RIT 2024-12-03T23:14:40,413 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733267710413 2024-12-03T23:14:40,413 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-03T23:14:40,414 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 256869bf1dc8aa33be79adb19e13d134, server=5f9d64419733,41167,1733267673270:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5f9d64419733,46833,1733267673070, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T23:14:40,414 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-03T23:14:40,415 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-03T23:14:40,416 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=742c43a49ea3b763b8232c8e8c8966b3, ASSIGN 2024-12-03T23:14:40,416 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T23:14:40,416 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=742c43a49ea3b763b8232c8e8c8966b3, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T23:14:40,418 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d9981bd2d541debb970a37ac121e7ed, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,418 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f71f90ef2d2f59e885f2ad6ca1a99596, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,418 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e35e57b6f6cbff01d3e782fbf6efd1f, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,418 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=55eff67791415c655cef877182b9bbe1, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,418 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=212a9963c9cddede26a4eb521cff5254, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,418 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f514b27fbd5c6ef451e11852713ea906, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,418 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=73cd6e72f89a0d14d1d979f76c9c07a4, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,418 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=256869bf1dc8aa33be79adb19e13d134, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,419 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2db7cef12ba09a51b0a7204fc81deeec, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,419 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=384b2e2791a8bf699cb7996e1aa7469c, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,419 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5ddac1aa9a6f0af338b83490d6d952c2, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,419 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1d026a7a0318e7deb79fa16b1262ae2c, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,419 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4e6706440af7674db97bb48a05858115, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,419 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f00ca5d778884039f3248393a629dcfc, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,419 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5e6925efcdb64a8de08b4c4bbf3a3d2e, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,419 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=742c43a49ea3b763b8232c8e8c8966b3, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,419 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=10beaf4047510d958e4a559194242a57, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,419 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=89a698454f10c9fb6cfc108c35d8415a, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,419 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b7da3736c6b257b43fe66ea13807bc6, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,419 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a6de975926600fad463e13238b2ac8a9, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,420 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e5e4ecc042f874ec5567f7e86548256f, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,420 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23a123891f32f583441cd7569a6de1f2, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,420 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0f09ea7cbe3ceec5232d11c21a512a4, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,420 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e96b12de28d6c3047803f54a3b027459, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,420 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=553e1b6e96399dd43d3b8f7cfad54903, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,420 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=db9924063978f5acb21b073c5d94a7ad, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,420 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-03T23:14:40,420 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733267740420 2024-12-03T23:14:40,420 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733267800420 2024-12-03T23:14:40,420 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 6 msec 2024-12-03T23:14:40,421 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-03T23:14:40,421 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T23:14:40,421 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-03T23:14:40,421 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-03T23:14:40,421 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-03T23:14:40,421 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-03T23:14:40,421 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-03T23:14:40,421 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-03T23:14:40,422 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-03T23:14:40,423 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-03T23:14:40,423 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T23:14:40,423 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T23:14:40,423 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T23:14:40,423 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T23:14:40,423 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T23:14:40,423 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-03T23:14:40,423 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-03T23:14:40,423 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T23:14:40,423 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-03T23:14:40,423 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-03T23:14:40,423 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-03T23:14:40,424 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-03T23:14:40,424 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-03T23:14:40,425 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 256869bf1dc8aa33be79adb19e13d134, server=5f9d64419733,41167,1733267673270:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5f9d64419733,46833,1733267673070, splitWal=true, meta=true 2024-12-03T23:14:40,425 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5f9d64419733%2C46833%2C1733267673070.meta.1733267675470.meta, worker=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,425 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5f9d64419733%2C46833%2C1733267673070.meta.1733267675470.meta, worker=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,425 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-03T23:14:40,425 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,425 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5f9d64419733%2C46833%2C1733267673070.1733267675041, worker=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,425 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5f9d64419733%2C46833%2C1733267673070.1733267675041, worker=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,425 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=23a123891f32f583441cd7569a6de1f2, ASSIGN 2024-12-03T23:14:40,425 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b7da3736c6b257b43fe66ea13807bc6, ASSIGN 2024-12-03T23:14:40,425 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e6706440af7674db97bb48a05858115, ASSIGN 2024-12-03T23:14:40,425 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=256869bf1dc8aa33be79adb19e13d134, ASSIGN 2024-12-03T23:14:40,425 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=742c43a49ea3b763b8232c8e8c8966b3, ASSIGN 2024-12-03T23:14:40,425 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 256869bf1dc8aa33be79adb19e13d134, server=5f9d64419733,41167,1733267673270:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5f9d64419733,46833,1733267673070, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,425 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5f9d64419733%2C46833%2C1733267673070.meta.1733267675470.meta, worker=5f9d64419733,41167,1733267673270 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,425 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5f9d64419733%2C46833%2C1733267673070.meta.1733267675470.meta, worker=5f9d64419733,41167,1733267673270 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,425 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,425 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=5f9d64419733,41167,1733267673270 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,425 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5f9d64419733%2C46833%2C1733267673070.1733267675041, worker=5f9d64419733,41167,1733267673270 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,425 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5f9d64419733%2C46833%2C1733267673070.1733267675041, worker=5f9d64419733,41167,1733267673270 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,425 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=23a123891f32f583441cd7569a6de1f2, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,425 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=23a123891f32f583441cd7569a6de1f2, ASSIGN 2024-12-03T23:14:40,425 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b7da3736c6b257b43fe66ea13807bc6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,425 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b7da3736c6b257b43fe66ea13807bc6, ASSIGN 2024-12-03T23:14:40,425 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e6706440af7674db97bb48a05858115, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,425 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e6706440af7674db97bb48a05858115, ASSIGN 2024-12-03T23:14:40,425 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=256869bf1dc8aa33be79adb19e13d134, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,425 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=256869bf1dc8aa33be79adb19e13d134, ASSIGN 2024-12-03T23:14:40,425 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=742c43a49ea3b763b8232c8e8c8966b3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,426 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=742c43a49ea3b763b8232c8e8c8966b3, ASSIGN 2024-12-03T23:14:40,426 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-12-03T23:14:40,426 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=61 2024-12-03T23:14:40,426 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=5f9d64419733,41167,1733267673270, table=hbase:meta, region=1588230740 2024-12-03T23:14:40,426 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 5f9d64419733,41167,1733267673270, state=OPEN 2024-12-03T23:14:40,428 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44899-0x1017a9bb4a60002, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T23:14:40,428 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41167-0x1017a9bb4a60003, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T23:14:40,428 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44117-0x1017a9bb4a60000, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T23:14:40,428 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T23:14:40,428 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T23:14:40,428 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T23:14:40,429 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=23a123891f32f583441cd7569a6de1f2, ASSIGN to state=OFFLINE, location=null, table=test, region=23a123891f32f583441cd7569a6de1f2 to restore RIT 2024-12-03T23:14:40,429 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b7da3736c6b257b43fe66ea13807bc6, ASSIGN to state=OFFLINE, location=null, table=test, region=2b7da3736c6b257b43fe66ea13807bc6 to restore RIT 2024-12-03T23:14:40,429 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e6706440af7674db97bb48a05858115, ASSIGN to state=OFFLINE, location=null, table=test, region=4e6706440af7674db97bb48a05858115 to restore RIT 2024-12-03T23:14:40,429 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=256869bf1dc8aa33be79adb19e13d134, ASSIGN to state=OFFLINE, location=null, table=test, region=256869bf1dc8aa33be79adb19e13d134 to restore RIT 2024-12-03T23:14:40,429 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=742c43a49ea3b763b8232c8e8c8966b3, ASSIGN to state=OFFLINE, location=null, table=test, region=742c43a49ea3b763b8232c8e8c8966b3 to restore RIT 2024-12-03T23:14:40,430 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733267710430 2024-12-03T23:14:40,430 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-03T23:14:40,430 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 256869bf1dc8aa33be79adb19e13d134, server=5f9d64419733,41167,1733267673270:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5f9d64419733,46833,1733267673070, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T23:14:40,431 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-03T23:14:40,433 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-03T23:14:40,433 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=256869bf1dc8aa33be79adb19e13d134, ASSIGN 2024-12-03T23:14:40,433 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T23:14:40,433 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=256869bf1dc8aa33be79adb19e13d134, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T23:14:40,435 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d9981bd2d541debb970a37ac121e7ed, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,436 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f71f90ef2d2f59e885f2ad6ca1a99596, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,436 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e35e57b6f6cbff01d3e782fbf6efd1f, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,436 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=55eff67791415c655cef877182b9bbe1, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,436 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=212a9963c9cddede26a4eb521cff5254, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,436 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f514b27fbd5c6ef451e11852713ea906, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,436 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=73cd6e72f89a0d14d1d979f76c9c07a4, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,437 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=256869bf1dc8aa33be79adb19e13d134, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,437 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2db7cef12ba09a51b0a7204fc81deeec, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,437 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=384b2e2791a8bf699cb7996e1aa7469c, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,437 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5ddac1aa9a6f0af338b83490d6d952c2, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,437 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1d026a7a0318e7deb79fa16b1262ae2c, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,437 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4e6706440af7674db97bb48a05858115, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,438 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f00ca5d778884039f3248393a629dcfc, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,438 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5e6925efcdb64a8de08b4c4bbf3a3d2e, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,438 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=742c43a49ea3b763b8232c8e8c8966b3, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,438 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=10beaf4047510d958e4a559194242a57, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,438 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=89a698454f10c9fb6cfc108c35d8415a, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,438 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b7da3736c6b257b43fe66ea13807bc6, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,438 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a6de975926600fad463e13238b2ac8a9, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,439 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e5e4ecc042f874ec5567f7e86548256f, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,439 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23a123891f32f583441cd7569a6de1f2, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,439 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0f09ea7cbe3ceec5232d11c21a512a4, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,439 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e96b12de28d6c3047803f54a3b027459, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,439 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=553e1b6e96399dd43d3b8f7cfad54903, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,439 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=db9924063978f5acb21b073c5d94a7ad, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,439 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-03T23:14:40,439 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733267740439 2024-12-03T23:14:40,440 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733267800440 2024-12-03T23:14:40,440 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-12-03T23:14:40,440 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-03T23:14:40,440 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T23:14:40,440 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-03T23:14:40,440 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-03T23:14:40,440 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-03T23:14:40,440 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-03T23:14:40,440 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-03T23:14:40,440 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44117 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T23:14:40,440 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-03T23:14:40,441 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-03T23:14:40,442 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-03T23:14:40,442 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T23:14:40,442 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T23:14:40,442 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T23:14:40,442 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T23:14:40,442 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-03T23:14:40,442 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-03T23:14:40,442 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T23:14:40,442 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-03T23:14:40,442 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-03T23:14:40,442 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-03T23:14:40,443 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-03T23:14:40,443 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-03T23:14:40,444 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 256869bf1dc8aa33be79adb19e13d134, server=5f9d64419733,41167,1733267673270:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5f9d64419733,46833,1733267673070, splitWal=true, meta=true 2024-12-03T23:14:40,444 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5f9d64419733%2C46833%2C1733267673070.meta.1733267675470.meta, worker=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,444 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5f9d64419733%2C46833%2C1733267673070.meta.1733267675470.meta, worker=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,444 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-03T23:14:40,444 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,444 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5f9d64419733%2C46833%2C1733267673070.1733267675041, worker=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,444 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5f9d64419733%2C46833%2C1733267673070.1733267675041, worker=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,444 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=23a123891f32f583441cd7569a6de1f2, ASSIGN 2024-12-03T23:14:40,444 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b7da3736c6b257b43fe66ea13807bc6, ASSIGN 2024-12-03T23:14:40,444 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e6706440af7674db97bb48a05858115, ASSIGN 2024-12-03T23:14:40,444 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=256869bf1dc8aa33be79adb19e13d134, ASSIGN 2024-12-03T23:14:40,444 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 256869bf1dc8aa33be79adb19e13d134, server=5f9d64419733,41167,1733267673270:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5f9d64419733,46833,1733267673070, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,444 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5f9d64419733%2C46833%2C1733267673070.meta.1733267675470.meta, worker=5f9d64419733,41167,1733267673270 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,444 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5f9d64419733%2C46833%2C1733267673070.meta.1733267675470.meta, worker=5f9d64419733,41167,1733267673270 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,444 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,444 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=5f9d64419733,41167,1733267673270 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,444 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5f9d64419733%2C46833%2C1733267673070.1733267675041, worker=5f9d64419733,41167,1733267673270 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,444 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5f9d64419733%2C46833%2C1733267673070.1733267675041, worker=5f9d64419733,41167,1733267673270 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,444 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=23a123891f32f583441cd7569a6de1f2, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,444 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=23a123891f32f583441cd7569a6de1f2, ASSIGN 2024-12-03T23:14:40,444 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b7da3736c6b257b43fe66ea13807bc6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,444 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b7da3736c6b257b43fe66ea13807bc6, ASSIGN 2024-12-03T23:14:40,444 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e6706440af7674db97bb48a05858115, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,444 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e6706440af7674db97bb48a05858115, ASSIGN 2024-12-03T23:14:40,444 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=256869bf1dc8aa33be79adb19e13d134, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,444 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=256869bf1dc8aa33be79adb19e13d134, ASSIGN 2024-12-03T23:14:40,444 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-12-03T23:14:40,445 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=61 2024-12-03T23:14:40,445 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=5f9d64419733,41167,1733267673270, table=hbase:meta, region=1588230740 2024-12-03T23:14:40,445 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 5f9d64419733,41167,1733267673270, state=OPEN 2024-12-03T23:14:40,447 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41167-0x1017a9bb4a60003, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T23:14:40,447 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44117-0x1017a9bb4a60000, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T23:14:40,447 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44899-0x1017a9bb4a60002, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T23:14:40,447 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T23:14:40,447 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T23:14:40,447 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T23:14:40,448 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=23a123891f32f583441cd7569a6de1f2, ASSIGN to state=OFFLINE, location=null, table=test, region=23a123891f32f583441cd7569a6de1f2 to restore RIT 2024-12-03T23:14:40,448 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b7da3736c6b257b43fe66ea13807bc6, ASSIGN to state=OFFLINE, location=null, table=test, region=2b7da3736c6b257b43fe66ea13807bc6 to restore RIT 2024-12-03T23:14:40,448 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e6706440af7674db97bb48a05858115, ASSIGN to state=OFFLINE, location=null, table=test, region=4e6706440af7674db97bb48a05858115 to restore RIT 2024-12-03T23:14:40,448 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=256869bf1dc8aa33be79adb19e13d134, ASSIGN to state=OFFLINE, location=null, table=test, region=256869bf1dc8aa33be79adb19e13d134 to restore RIT 2024-12-03T23:14:40,449 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733267710449 2024-12-03T23:14:40,449 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-03T23:14:40,449 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 256869bf1dc8aa33be79adb19e13d134, server=5f9d64419733,41167,1733267673270:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5f9d64419733,46833,1733267673070, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T23:14:40,450 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-03T23:14:40,451 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-03T23:14:40,451 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e6706440af7674db97bb48a05858115, ASSIGN 2024-12-03T23:14:40,451 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T23:14:40,451 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e6706440af7674db97bb48a05858115, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T23:14:40,454 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d9981bd2d541debb970a37ac121e7ed, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,454 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f71f90ef2d2f59e885f2ad6ca1a99596, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,454 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e35e57b6f6cbff01d3e782fbf6efd1f, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,455 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=55eff67791415c655cef877182b9bbe1, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,455 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=212a9963c9cddede26a4eb521cff5254, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,455 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f514b27fbd5c6ef451e11852713ea906, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,455 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=73cd6e72f89a0d14d1d979f76c9c07a4, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,455 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=256869bf1dc8aa33be79adb19e13d134, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,455 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2db7cef12ba09a51b0a7204fc81deeec, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,455 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=384b2e2791a8bf699cb7996e1aa7469c, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,455 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5ddac1aa9a6f0af338b83490d6d952c2, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,455 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1d026a7a0318e7deb79fa16b1262ae2c, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,455 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4e6706440af7674db97bb48a05858115, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,456 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f00ca5d778884039f3248393a629dcfc, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,456 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5e6925efcdb64a8de08b4c4bbf3a3d2e, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,456 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=742c43a49ea3b763b8232c8e8c8966b3, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,456 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=10beaf4047510d958e4a559194242a57, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,456 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=89a698454f10c9fb6cfc108c35d8415a, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,456 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b7da3736c6b257b43fe66ea13807bc6, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,456 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a6de975926600fad463e13238b2ac8a9, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,456 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e5e4ecc042f874ec5567f7e86548256f, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,456 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23a123891f32f583441cd7569a6de1f2, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,456 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0f09ea7cbe3ceec5232d11c21a512a4, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,456 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e96b12de28d6c3047803f54a3b027459, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,456 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=553e1b6e96399dd43d3b8f7cfad54903, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,457 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=db9924063978f5acb21b073c5d94a7ad, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,457 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-03T23:14:40,457 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733267740457 2024-12-03T23:14:40,457 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733267800457 2024-12-03T23:14:40,457 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-12-03T23:14:40,457 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-03T23:14:40,457 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T23:14:40,457 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-03T23:14:40,457 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-03T23:14:40,457 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-03T23:14:40,457 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-03T23:14:40,457 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-03T23:14:40,457 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-03T23:14:40,458 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-03T23:14:40,459 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-03T23:14:40,459 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T23:14:40,459 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T23:14:40,459 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T23:14:40,459 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-03T23:14:40,460 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-03T23:14:40,460 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T23:14:40,460 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-03T23:14:40,460 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-03T23:14:40,460 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-03T23:14:40,460 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-03T23:14:40,460 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-03T23:14:40,461 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 256869bf1dc8aa33be79adb19e13d134, server=5f9d64419733,41167,1733267673270:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5f9d64419733,46833,1733267673070, splitWal=true, meta=true 2024-12-03T23:14:40,461 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5f9d64419733%2C46833%2C1733267673070.meta.1733267675470.meta, worker=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,461 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5f9d64419733%2C46833%2C1733267673070.meta.1733267675470.meta, worker=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,461 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-03T23:14:40,461 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,461 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5f9d64419733%2C46833%2C1733267673070.1733267675041, worker=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,461 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5f9d64419733%2C46833%2C1733267673070.1733267675041, worker=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,461 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=23a123891f32f583441cd7569a6de1f2, ASSIGN 2024-12-03T23:14:40,461 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b7da3736c6b257b43fe66ea13807bc6, ASSIGN 2024-12-03T23:14:40,461 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e6706440af7674db97bb48a05858115, ASSIGN 2024-12-03T23:14:40,461 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 256869bf1dc8aa33be79adb19e13d134, server=5f9d64419733,41167,1733267673270:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5f9d64419733,46833,1733267673070, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,461 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5f9d64419733%2C46833%2C1733267673070.meta.1733267675470.meta, worker=5f9d64419733,41167,1733267673270 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,461 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5f9d64419733%2C46833%2C1733267673070.meta.1733267675470.meta, worker=5f9d64419733,41167,1733267673270 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,461 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,461 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=5f9d64419733,41167,1733267673270 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,461 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5f9d64419733%2C46833%2C1733267673070.1733267675041, worker=5f9d64419733,41167,1733267673270 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,461 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5f9d64419733%2C46833%2C1733267673070.1733267675041, worker=5f9d64419733,41167,1733267673270 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,461 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=23a123891f32f583441cd7569a6de1f2, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,462 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=23a123891f32f583441cd7569a6de1f2, ASSIGN 2024-12-03T23:14:40,462 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b7da3736c6b257b43fe66ea13807bc6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,462 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b7da3736c6b257b43fe66ea13807bc6, ASSIGN 2024-12-03T23:14:40,462 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e6706440af7674db97bb48a05858115, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,462 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e6706440af7674db97bb48a05858115, ASSIGN 2024-12-03T23:14:40,462 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-12-03T23:14:40,462 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=61 2024-12-03T23:14:40,462 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=5f9d64419733,41167,1733267673270, table=hbase:meta, region=1588230740 2024-12-03T23:14:40,463 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 5f9d64419733,41167,1733267673270, state=OPEN 2024-12-03T23:14:40,465 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44117-0x1017a9bb4a60000, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T23:14:40,465 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44899-0x1017a9bb4a60002, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T23:14:40,465 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41167-0x1017a9bb4a60003, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T23:14:40,465 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T23:14:40,465 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T23:14:40,465 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T23:14:40,465 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=23a123891f32f583441cd7569a6de1f2, ASSIGN to state=OFFLINE, location=null, table=test, region=23a123891f32f583441cd7569a6de1f2 to restore RIT 2024-12-03T23:14:40,465 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b7da3736c6b257b43fe66ea13807bc6, ASSIGN to state=OFFLINE, location=null, table=test, region=2b7da3736c6b257b43fe66ea13807bc6 to restore RIT 2024-12-03T23:14:40,465 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e6706440af7674db97bb48a05858115, ASSIGN to state=OFFLINE, location=null, table=test, region=4e6706440af7674db97bb48a05858115 to restore RIT 2024-12-03T23:14:40,466 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733267710466 2024-12-03T23:14:40,466 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-03T23:14:40,466 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 256869bf1dc8aa33be79adb19e13d134, server=5f9d64419733,41167,1733267673270:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5f9d64419733,46833,1733267673070, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T23:14:40,467 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-03T23:14:40,468 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-03T23:14:40,468 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b7da3736c6b257b43fe66ea13807bc6, ASSIGN 2024-12-03T23:14:40,468 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T23:14:40,468 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b7da3736c6b257b43fe66ea13807bc6, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T23:14:40,470 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d9981bd2d541debb970a37ac121e7ed, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,470 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f71f90ef2d2f59e885f2ad6ca1a99596, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,470 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e35e57b6f6cbff01d3e782fbf6efd1f, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,471 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=55eff67791415c655cef877182b9bbe1, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,471 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=212a9963c9cddede26a4eb521cff5254, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,471 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f514b27fbd5c6ef451e11852713ea906, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,471 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=73cd6e72f89a0d14d1d979f76c9c07a4, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,471 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=256869bf1dc8aa33be79adb19e13d134, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,471 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2db7cef12ba09a51b0a7204fc81deeec, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,472 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=384b2e2791a8bf699cb7996e1aa7469c, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,472 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5ddac1aa9a6f0af338b83490d6d952c2, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,472 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1d026a7a0318e7deb79fa16b1262ae2c, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,472 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4e6706440af7674db97bb48a05858115, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,472 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f00ca5d778884039f3248393a629dcfc, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,472 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5e6925efcdb64a8de08b4c4bbf3a3d2e, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,472 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=742c43a49ea3b763b8232c8e8c8966b3, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,473 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=10beaf4047510d958e4a559194242a57, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,473 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=89a698454f10c9fb6cfc108c35d8415a, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,473 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b7da3736c6b257b43fe66ea13807bc6, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,473 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a6de975926600fad463e13238b2ac8a9, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,473 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e5e4ecc042f874ec5567f7e86548256f, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,473 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23a123891f32f583441cd7569a6de1f2, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,473 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0f09ea7cbe3ceec5232d11c21a512a4, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,474 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e96b12de28d6c3047803f54a3b027459, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,474 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=553e1b6e96399dd43d3b8f7cfad54903, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,474 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=db9924063978f5acb21b073c5d94a7ad, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,474 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-03T23:14:40,474 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733267740474 2024-12-03T23:14:40,474 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733267800474 2024-12-03T23:14:40,474 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-12-03T23:14:40,474 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-03T23:14:40,474 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T23:14:40,474 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-03T23:14:40,474 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-03T23:14:40,475 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-03T23:14:40,475 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-03T23:14:40,475 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-03T23:14:40,475 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-03T23:14:40,476 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-03T23:14:40,477 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-03T23:14:40,478 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T23:14:40,478 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T23:14:40,478 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-03T23:14:40,478 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-03T23:14:40,478 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T23:14:40,478 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-03T23:14:40,478 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-03T23:14:40,478 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-03T23:14:40,478 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-03T23:14:40,479 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-03T23:14:40,480 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 256869bf1dc8aa33be79adb19e13d134, server=5f9d64419733,41167,1733267673270:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5f9d64419733,46833,1733267673070, splitWal=true, meta=true 2024-12-03T23:14:40,480 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5f9d64419733%2C46833%2C1733267673070.meta.1733267675470.meta, worker=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,480 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5f9d64419733%2C46833%2C1733267673070.meta.1733267675470.meta, worker=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,480 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-03T23:14:40,480 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,480 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5f9d64419733%2C46833%2C1733267673070.1733267675041, worker=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,480 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5f9d64419733%2C46833%2C1733267673070.1733267675041, worker=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,480 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=23a123891f32f583441cd7569a6de1f2, ASSIGN 2024-12-03T23:14:40,480 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b7da3736c6b257b43fe66ea13807bc6, ASSIGN 2024-12-03T23:14:40,480 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 256869bf1dc8aa33be79adb19e13d134, server=5f9d64419733,41167,1733267673270:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5f9d64419733,46833,1733267673070, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,480 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5f9d64419733%2C46833%2C1733267673070.meta.1733267675470.meta, worker=5f9d64419733,41167,1733267673270 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,480 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5f9d64419733%2C46833%2C1733267673070.meta.1733267675470.meta, worker=5f9d64419733,41167,1733267673270 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,480 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,480 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=5f9d64419733,41167,1733267673270 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,480 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5f9d64419733%2C46833%2C1733267673070.1733267675041, worker=5f9d64419733,41167,1733267673270 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,480 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5f9d64419733%2C46833%2C1733267673070.1733267675041, worker=5f9d64419733,41167,1733267673270 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,480 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=23a123891f32f583441cd7569a6de1f2, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,480 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=23a123891f32f583441cd7569a6de1f2, ASSIGN 2024-12-03T23:14:40,481 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b7da3736c6b257b43fe66ea13807bc6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,481 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b7da3736c6b257b43fe66ea13807bc6, ASSIGN 2024-12-03T23:14:40,481 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-12-03T23:14:40,481 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=61 2024-12-03T23:14:40,481 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=5f9d64419733,41167,1733267673270, table=hbase:meta, region=1588230740 2024-12-03T23:14:40,482 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 5f9d64419733,41167,1733267673270, state=OPEN 2024-12-03T23:14:40,491 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41167-0x1017a9bb4a60003, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T23:14:40,491 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44117-0x1017a9bb4a60000, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T23:14:40,491 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44899-0x1017a9bb4a60002, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T23:14:40,491 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T23:14:40,491 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T23:14:40,491 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T23:14:40,492 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=23a123891f32f583441cd7569a6de1f2, ASSIGN to state=OFFLINE, location=null, table=test, region=23a123891f32f583441cd7569a6de1f2 to restore RIT 2024-12-03T23:14:40,492 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b7da3736c6b257b43fe66ea13807bc6, ASSIGN to state=OFFLINE, location=null, table=test, region=2b7da3736c6b257b43fe66ea13807bc6 to restore RIT 2024-12-03T23:14:40,492 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733267710492 2024-12-03T23:14:40,492 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-03T23:14:40,492 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 256869bf1dc8aa33be79adb19e13d134, server=5f9d64419733,41167,1733267673270:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5f9d64419733,46833,1733267673070, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T23:14:40,494 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-03T23:14:40,495 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-03T23:14:40,495 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=23a123891f32f583441cd7569a6de1f2, ASSIGN 2024-12-03T23:14:40,495 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T23:14:40,495 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=23a123891f32f583441cd7569a6de1f2, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T23:14:40,497 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d9981bd2d541debb970a37ac121e7ed, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,497 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f71f90ef2d2f59e885f2ad6ca1a99596, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,497 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e35e57b6f6cbff01d3e782fbf6efd1f, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,497 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=55eff67791415c655cef877182b9bbe1, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,497 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=212a9963c9cddede26a4eb521cff5254, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,498 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f514b27fbd5c6ef451e11852713ea906, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,498 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=73cd6e72f89a0d14d1d979f76c9c07a4, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,498 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=256869bf1dc8aa33be79adb19e13d134, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,498 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2db7cef12ba09a51b0a7204fc81deeec, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,498 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=384b2e2791a8bf699cb7996e1aa7469c, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,498 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5ddac1aa9a6f0af338b83490d6d952c2, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,498 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1d026a7a0318e7deb79fa16b1262ae2c, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,499 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4e6706440af7674db97bb48a05858115, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,499 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f00ca5d778884039f3248393a629dcfc, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,499 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5e6925efcdb64a8de08b4c4bbf3a3d2e, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,499 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=742c43a49ea3b763b8232c8e8c8966b3, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,499 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=10beaf4047510d958e4a559194242a57, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,499 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=89a698454f10c9fb6cfc108c35d8415a, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,500 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b7da3736c6b257b43fe66ea13807bc6, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,500 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a6de975926600fad463e13238b2ac8a9, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,500 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e5e4ecc042f874ec5567f7e86548256f, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,500 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23a123891f32f583441cd7569a6de1f2, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,500 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0f09ea7cbe3ceec5232d11c21a512a4, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,500 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e96b12de28d6c3047803f54a3b027459, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,500 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=553e1b6e96399dd43d3b8f7cfad54903, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,501 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=db9924063978f5acb21b073c5d94a7ad, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,501 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-03T23:14:40,501 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733267740501 2024-12-03T23:14:40,501 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733267800501 2024-12-03T23:14:40,501 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-12-03T23:14:40,501 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-03T23:14:40,501 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T23:14:40,501 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-03T23:14:40,501 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-03T23:14:40,501 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-03T23:14:40,501 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-03T23:14:40,501 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-03T23:14:40,502 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-03T23:14:40,502 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-03T23:14:40,504 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-03T23:14:40,504 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T23:14:40,504 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-03T23:14:40,505 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-03T23:14:40,505 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T23:14:40,505 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-03T23:14:40,505 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-03T23:14:40,505 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-03T23:14:40,505 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-03T23:14:40,506 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-03T23:14:40,507 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 256869bf1dc8aa33be79adb19e13d134, server=5f9d64419733,41167,1733267673270:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5f9d64419733,46833,1733267673070, splitWal=true, meta=true 2024-12-03T23:14:40,507 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5f9d64419733%2C46833%2C1733267673070.meta.1733267675470.meta, worker=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,507 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5f9d64419733%2C46833%2C1733267673070.meta.1733267675470.meta, worker=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,507 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-03T23:14:40,507 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,508 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5f9d64419733%2C46833%2C1733267673070.1733267675041, worker=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,508 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5f9d64419733%2C46833%2C1733267673070.1733267675041, worker=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,508 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=23a123891f32f583441cd7569a6de1f2, ASSIGN 2024-12-03T23:14:40,508 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 256869bf1dc8aa33be79adb19e13d134, server=5f9d64419733,41167,1733267673270:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5f9d64419733,46833,1733267673070, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,508 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5f9d64419733%2C46833%2C1733267673070.meta.1733267675470.meta, worker=5f9d64419733,41167,1733267673270 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,508 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5f9d64419733%2C46833%2C1733267673070.meta.1733267675470.meta, worker=5f9d64419733,41167,1733267673270 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,508 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,508 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=5f9d64419733,41167,1733267673270 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,508 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5f9d64419733%2C46833%2C1733267673070.1733267675041, worker=5f9d64419733,41167,1733267673270 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,508 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5f9d64419733%2C46833%2C1733267673070.1733267675041, worker=5f9d64419733,41167,1733267673270 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,508 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=23a123891f32f583441cd7569a6de1f2, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,508 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=23a123891f32f583441cd7569a6de1f2, ASSIGN 2024-12-03T23:14:40,508 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-12-03T23:14:40,509 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=61 2024-12-03T23:14:40,509 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=5f9d64419733,41167,1733267673270, table=hbase:meta, region=1588230740 2024-12-03T23:14:40,509 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 5f9d64419733,41167,1733267673270, state=OPEN 2024-12-03T23:14:40,511 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44899-0x1017a9bb4a60002, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T23:14:40,511 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44117-0x1017a9bb4a60000, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T23:14:40,511 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41167-0x1017a9bb4a60003, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T23:14:40,512 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T23:14:40,512 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T23:14:40,512 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T23:14:40,512 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=23a123891f32f583441cd7569a6de1f2, ASSIGN to state=OFFLINE, location=null, table=test, region=23a123891f32f583441cd7569a6de1f2 to restore RIT 2024-12-03T23:14:40,513 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733267710513 2024-12-03T23:14:40,513 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-03T23:14:40,513 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 256869bf1dc8aa33be79adb19e13d134, server=5f9d64419733,41167,1733267673270:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5f9d64419733,46833,1733267673070, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T23:14:40,514 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-03T23:14:40,516 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-03T23:14:40,516 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5f9d64419733%2C46833%2C1733267673070.1733267675041, worker=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,516 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T23:14:40,516 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5f9d64419733%2C46833%2C1733267673070.1733267675041, worker=5f9d64419733,41167,1733267673270 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T23:14:40,519 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d9981bd2d541debb970a37ac121e7ed, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,519 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f71f90ef2d2f59e885f2ad6ca1a99596, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,519 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e35e57b6f6cbff01d3e782fbf6efd1f, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,519 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=55eff67791415c655cef877182b9bbe1, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,519 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=212a9963c9cddede26a4eb521cff5254, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,519 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f514b27fbd5c6ef451e11852713ea906, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,519 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=73cd6e72f89a0d14d1d979f76c9c07a4, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,519 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=256869bf1dc8aa33be79adb19e13d134, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,520 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2db7cef12ba09a51b0a7204fc81deeec, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,520 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=384b2e2791a8bf699cb7996e1aa7469c, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,520 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5ddac1aa9a6f0af338b83490d6d952c2, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,520 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1d026a7a0318e7deb79fa16b1262ae2c, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,520 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4e6706440af7674db97bb48a05858115, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,520 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f00ca5d778884039f3248393a629dcfc, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,521 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5e6925efcdb64a8de08b4c4bbf3a3d2e, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,521 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=742c43a49ea3b763b8232c8e8c8966b3, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,521 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=10beaf4047510d958e4a559194242a57, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,521 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=89a698454f10c9fb6cfc108c35d8415a, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,521 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b7da3736c6b257b43fe66ea13807bc6, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,521 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a6de975926600fad463e13238b2ac8a9, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,521 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e5e4ecc042f874ec5567f7e86548256f, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,521 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23a123891f32f583441cd7569a6de1f2, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,521 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0f09ea7cbe3ceec5232d11c21a512a4, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,522 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e96b12de28d6c3047803f54a3b027459, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,522 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=553e1b6e96399dd43d3b8f7cfad54903, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,522 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=db9924063978f5acb21b073c5d94a7ad, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,522 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-03T23:14:40,522 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733267740522 2024-12-03T23:14:40,522 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733267800522 2024-12-03T23:14:40,522 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-12-03T23:14:40,522 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-03T23:14:40,522 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T23:14:40,522 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-03T23:14:40,522 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-03T23:14:40,522 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-03T23:14:40,523 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-03T23:14:40,523 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-03T23:14:40,523 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-03T23:14:40,524 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-03T23:14:40,526 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-03T23:14:40,526 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-03T23:14:40,526 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-03T23:14:40,526 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T23:14:40,526 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-03T23:14:40,526 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-03T23:14:40,526 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-03T23:14:40,526 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-03T23:14:40,527 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-03T23:14:40,528 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 256869bf1dc8aa33be79adb19e13d134, server=5f9d64419733,41167,1733267673270:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5f9d64419733,46833,1733267673070, splitWal=true, meta=true 2024-12-03T23:14:40,528 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5f9d64419733%2C46833%2C1733267673070.meta.1733267675470.meta, worker=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,528 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5f9d64419733%2C46833%2C1733267673070.meta.1733267675470.meta, worker=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,528 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-03T23:14:40,528 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,528 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5f9d64419733%2C46833%2C1733267673070.1733267675041, worker=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,528 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5f9d64419733%2C46833%2C1733267673070.1733267675041, worker=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,528 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 256869bf1dc8aa33be79adb19e13d134, server=5f9d64419733,41167,1733267673270:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5f9d64419733,46833,1733267673070, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,528 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5f9d64419733%2C46833%2C1733267673070.meta.1733267675470.meta, worker=5f9d64419733,41167,1733267673270 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,528 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5f9d64419733%2C46833%2C1733267673070.meta.1733267675470.meta, worker=5f9d64419733,41167,1733267673270 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,528 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,528 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=5f9d64419733,41167,1733267673270 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,528 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5f9d64419733%2C46833%2C1733267673070.1733267675041, worker=5f9d64419733,41167,1733267673270 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,528 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5f9d64419733%2C46833%2C1733267673070.1733267675041, worker=5f9d64419733,41167,1733267673270 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,528 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-12-03T23:14:40,529 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=61 2024-12-03T23:14:40,529 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=5f9d64419733,41167,1733267673270, table=hbase:meta, region=1588230740 2024-12-03T23:14:40,529 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 5f9d64419733,41167,1733267673270, state=OPEN 2024-12-03T23:14:40,531 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41167-0x1017a9bb4a60003, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T23:14:40,531 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44117-0x1017a9bb4a60000, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T23:14:40,531 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T23:14:40,531 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T23:14:40,532 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44899-0x1017a9bb4a60002, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T23:14:40,532 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T23:14:40,533 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733267710533 2024-12-03T23:14:40,533 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-03T23:14:40,533 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 256869bf1dc8aa33be79adb19e13d134, server=5f9d64419733,41167,1733267673270:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5f9d64419733,46833,1733267673070, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T23:14:40,534 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-03T23:14:40,536 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-03T23:14:40,536 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5f9d64419733%2C46833%2C1733267673070.1733267675041, worker=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,536 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T23:14:40,536 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5f9d64419733%2C46833%2C1733267673070.1733267675041, worker=5f9d64419733,41167,1733267673270 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T23:14:40,537 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d9981bd2d541debb970a37ac121e7ed, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,537 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f71f90ef2d2f59e885f2ad6ca1a99596, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,537 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e35e57b6f6cbff01d3e782fbf6efd1f, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,538 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=55eff67791415c655cef877182b9bbe1, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,538 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=212a9963c9cddede26a4eb521cff5254, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,538 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f514b27fbd5c6ef451e11852713ea906, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,538 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=73cd6e72f89a0d14d1d979f76c9c07a4, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,538 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=256869bf1dc8aa33be79adb19e13d134, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,538 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2db7cef12ba09a51b0a7204fc81deeec, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,538 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=384b2e2791a8bf699cb7996e1aa7469c, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,538 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5ddac1aa9a6f0af338b83490d6d952c2, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,538 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1d026a7a0318e7deb79fa16b1262ae2c, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,538 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4e6706440af7674db97bb48a05858115, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,538 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f00ca5d778884039f3248393a629dcfc, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,539 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5e6925efcdb64a8de08b4c4bbf3a3d2e, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,539 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=742c43a49ea3b763b8232c8e8c8966b3, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,539 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=10beaf4047510d958e4a559194242a57, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,539 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=89a698454f10c9fb6cfc108c35d8415a, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,539 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b7da3736c6b257b43fe66ea13807bc6, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,539 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a6de975926600fad463e13238b2ac8a9, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,539 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e5e4ecc042f874ec5567f7e86548256f, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,539 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23a123891f32f583441cd7569a6de1f2, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,539 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0f09ea7cbe3ceec5232d11c21a512a4, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,539 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e96b12de28d6c3047803f54a3b027459, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,539 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=553e1b6e96399dd43d3b8f7cfad54903, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,539 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=db9924063978f5acb21b073c5d94a7ad, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,539 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-03T23:14:40,539 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733267740539 2024-12-03T23:14:40,539 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733267800539 2024-12-03T23:14:40,539 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 6 msec 2024-12-03T23:14:40,539 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-03T23:14:40,539 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T23:14:40,540 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-03T23:14:40,540 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-03T23:14:40,540 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-03T23:14:40,540 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-03T23:14:40,540 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-03T23:14:40,540 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-03T23:14:40,541 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-03T23:14:40,542 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-03T23:14:40,542 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-03T23:14:40,542 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-03T23:14:40,542 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T23:14:40,542 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-03T23:14:40,542 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-03T23:14:40,542 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-03T23:14:40,543 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-03T23:14:40,543 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44117 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T23:14:40,543 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 256869bf1dc8aa33be79adb19e13d134, server=5f9d64419733,41167,1733267673270:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5f9d64419733,46833,1733267673070, splitWal=true, meta=true 2024-12-03T23:14:40,543 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5f9d64419733%2C46833%2C1733267673070.meta.1733267675470.meta, worker=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,543 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5f9d64419733%2C46833%2C1733267673070.meta.1733267675470.meta, worker=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,543 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-03T23:14:40,543 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,544 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5f9d64419733%2C46833%2C1733267673070.1733267675041, worker=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,544 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 256869bf1dc8aa33be79adb19e13d134, server=5f9d64419733,41167,1733267673270:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5f9d64419733,46833,1733267673070, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,544 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5f9d64419733%2C46833%2C1733267673070.meta.1733267675470.meta, worker=5f9d64419733,41167,1733267673270 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,544 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5f9d64419733%2C46833%2C1733267673070.meta.1733267675470.meta, worker=5f9d64419733,41167,1733267673270 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,544 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,544 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=5f9d64419733,41167,1733267673270 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,544 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5f9d64419733%2C46833%2C1733267673070.1733267675041, worker=5f9d64419733,41167,1733267673270 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,544 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-12-03T23:14:40,545 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=61 2024-12-03T23:14:40,545 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=5f9d64419733,41167,1733267673270, table=hbase:meta, region=1588230740 2024-12-03T23:14:40,545 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 5f9d64419733,41167,1733267673270, state=OPEN 2024-12-03T23:14:40,547 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41167-0x1017a9bb4a60003, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T23:14:40,547 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44899-0x1017a9bb4a60002, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T23:14:40,547 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44117-0x1017a9bb4a60000, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T23:14:40,547 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T23:14:40,547 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T23:14:40,547 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T23:14:40,550 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733267710550 2024-12-03T23:14:40,550 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-03T23:14:40,550 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 256869bf1dc8aa33be79adb19e13d134, server=5f9d64419733,41167,1733267673270:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5f9d64419733,46833,1733267673070, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T23:14:40,551 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-03T23:14:40,553 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-03T23:14:40,553 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,553 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T23:14:40,554 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=5f9d64419733,41167,1733267673270 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T23:14:40,555 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d9981bd2d541debb970a37ac121e7ed, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,555 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f71f90ef2d2f59e885f2ad6ca1a99596, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,555 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e35e57b6f6cbff01d3e782fbf6efd1f, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,555 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=55eff67791415c655cef877182b9bbe1, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,556 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=212a9963c9cddede26a4eb521cff5254, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,556 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f514b27fbd5c6ef451e11852713ea906, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,556 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=73cd6e72f89a0d14d1d979f76c9c07a4, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,556 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=256869bf1dc8aa33be79adb19e13d134, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,556 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2db7cef12ba09a51b0a7204fc81deeec, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,556 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=384b2e2791a8bf699cb7996e1aa7469c, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,556 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5ddac1aa9a6f0af338b83490d6d952c2, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,556 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1d026a7a0318e7deb79fa16b1262ae2c, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,556 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4e6706440af7674db97bb48a05858115, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,556 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f00ca5d778884039f3248393a629dcfc, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,556 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5e6925efcdb64a8de08b4c4bbf3a3d2e, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,556 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=742c43a49ea3b763b8232c8e8c8966b3, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,557 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=10beaf4047510d958e4a559194242a57, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,557 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=89a698454f10c9fb6cfc108c35d8415a, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,557 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b7da3736c6b257b43fe66ea13807bc6, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,557 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a6de975926600fad463e13238b2ac8a9, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,557 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e5e4ecc042f874ec5567f7e86548256f, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,557 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23a123891f32f583441cd7569a6de1f2, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,557 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0f09ea7cbe3ceec5232d11c21a512a4, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,557 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e96b12de28d6c3047803f54a3b027459, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,557 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=553e1b6e96399dd43d3b8f7cfad54903, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,557 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=db9924063978f5acb21b073c5d94a7ad, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,557 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-03T23:14:40,557 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733267740557 2024-12-03T23:14:40,557 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733267800557 2024-12-03T23:14:40,557 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-12-03T23:14:40,557 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-03T23:14:40,557 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T23:14:40,557 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-03T23:14:40,557 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-03T23:14:40,558 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-03T23:14:40,558 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-03T23:14:40,558 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-03T23:14:40,558 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-03T23:14:40,559 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-03T23:14:40,560 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-03T23:14:40,560 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-03T23:14:40,560 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-03T23:14:40,560 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T23:14:40,560 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-03T23:14:40,560 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-03T23:14:40,561 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-03T23:14:40,561 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 256869bf1dc8aa33be79adb19e13d134, server=5f9d64419733,41167,1733267673270:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5f9d64419733,46833,1733267673070, splitWal=true, meta=true 2024-12-03T23:14:40,561 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5f9d64419733%2C46833%2C1733267673070.meta.1733267675470.meta, worker=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,561 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5f9d64419733%2C46833%2C1733267673070.meta.1733267675470.meta, worker=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,561 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-03T23:14:40,561 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,561 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 256869bf1dc8aa33be79adb19e13d134, server=5f9d64419733,41167,1733267673270:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5f9d64419733,46833,1733267673070, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,561 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5f9d64419733%2C46833%2C1733267673070.meta.1733267675470.meta, worker=5f9d64419733,41167,1733267673270 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,561 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5f9d64419733%2C46833%2C1733267673070.meta.1733267675470.meta, worker=5f9d64419733,41167,1733267673270 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,561 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,561 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=5f9d64419733,41167,1733267673270 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,561 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-12-03T23:14:40,562 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=61 2024-12-03T23:14:40,562 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=5f9d64419733,41167,1733267673270, table=hbase:meta, region=1588230740 2024-12-03T23:14:40,562 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 5f9d64419733,41167,1733267673270, state=OPEN 2024-12-03T23:14:40,565 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41167-0x1017a9bb4a60003, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T23:14:40,565 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44899-0x1017a9bb4a60002, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T23:14:40,565 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44117-0x1017a9bb4a60000, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T23:14:40,565 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T23:14:40,565 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T23:14:40,565 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T23:14:40,566 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733267710566 2024-12-03T23:14:40,566 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-03T23:14:40,566 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 256869bf1dc8aa33be79adb19e13d134, server=5f9d64419733,41167,1733267673270:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5f9d64419733,46833,1733267673070, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T23:14:40,568 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-03T23:14:40,569 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-03T23:14:40,569 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-03T23:14:40,569 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T23:14:40,569 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T23:14:40,571 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d9981bd2d541debb970a37ac121e7ed, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,571 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f71f90ef2d2f59e885f2ad6ca1a99596, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,571 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e35e57b6f6cbff01d3e782fbf6efd1f, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,572 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=55eff67791415c655cef877182b9bbe1, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,572 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=212a9963c9cddede26a4eb521cff5254, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,572 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f514b27fbd5c6ef451e11852713ea906, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,572 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=73cd6e72f89a0d14d1d979f76c9c07a4, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,572 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=256869bf1dc8aa33be79adb19e13d134, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,572 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2db7cef12ba09a51b0a7204fc81deeec, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,572 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=384b2e2791a8bf699cb7996e1aa7469c, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,572 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5ddac1aa9a6f0af338b83490d6d952c2, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,572 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1d026a7a0318e7deb79fa16b1262ae2c, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,572 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4e6706440af7674db97bb48a05858115, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,572 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f00ca5d778884039f3248393a629dcfc, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,572 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5e6925efcdb64a8de08b4c4bbf3a3d2e, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,572 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=742c43a49ea3b763b8232c8e8c8966b3, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,572 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=10beaf4047510d958e4a559194242a57, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,572 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=89a698454f10c9fb6cfc108c35d8415a, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,573 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b7da3736c6b257b43fe66ea13807bc6, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,573 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a6de975926600fad463e13238b2ac8a9, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,573 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e5e4ecc042f874ec5567f7e86548256f, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,573 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23a123891f32f583441cd7569a6de1f2, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,573 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0f09ea7cbe3ceec5232d11c21a512a4, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,573 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e96b12de28d6c3047803f54a3b027459, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,573 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=553e1b6e96399dd43d3b8f7cfad54903, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,573 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=db9924063978f5acb21b073c5d94a7ad, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,573 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-03T23:14:40,573 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733267740573 2024-12-03T23:14:40,573 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733267800573 2024-12-03T23:14:40,573 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 6 msec 2024-12-03T23:14:40,573 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-03T23:14:40,573 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T23:14:40,573 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-03T23:14:40,573 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-03T23:14:40,574 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-03T23:14:40,574 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-03T23:14:40,574 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-03T23:14:40,574 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-03T23:14:40,575 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-03T23:14:40,576 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-03T23:14:40,576 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-03T23:14:40,576 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-03T23:14:40,576 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T23:14:40,576 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-03T23:14:40,576 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-03T23:14:40,577 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 256869bf1dc8aa33be79adb19e13d134, server=5f9d64419733,41167,1733267673270:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5f9d64419733,46833,1733267673070, splitWal=true, meta=true 2024-12-03T23:14:40,577 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5f9d64419733%2C46833%2C1733267673070.meta.1733267675470.meta, worker=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,577 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5f9d64419733%2C46833%2C1733267673070.meta.1733267675470.meta, worker=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,577 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-03T23:14:40,577 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 256869bf1dc8aa33be79adb19e13d134, server=5f9d64419733,41167,1733267673270:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5f9d64419733,46833,1733267673070, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,577 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5f9d64419733%2C46833%2C1733267673070.meta.1733267675470.meta, worker=5f9d64419733,41167,1733267673270 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,577 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5f9d64419733%2C46833%2C1733267673070.meta.1733267675470.meta, worker=5f9d64419733,41167,1733267673270 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,577 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,577 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-12-03T23:14:40,578 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=61 2024-12-03T23:14:40,578 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=5f9d64419733,41167,1733267673270, table=hbase:meta, region=1588230740 2024-12-03T23:14:40,578 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 5f9d64419733,41167,1733267673270, state=OPEN 2024-12-03T23:14:40,580 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41167-0x1017a9bb4a60003, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T23:14:40,580 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44899-0x1017a9bb4a60002, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T23:14:40,580 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44117-0x1017a9bb4a60000, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T23:14:40,580 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T23:14:40,580 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T23:14:40,580 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T23:14:40,581 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733267710581 2024-12-03T23:14:40,581 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-03T23:14:40,581 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 256869bf1dc8aa33be79adb19e13d134, server=5f9d64419733,41167,1733267673270:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5f9d64419733,46833,1733267673070, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T23:14:40,582 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-03T23:14:40,584 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-03T23:14:40,584 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5f9d64419733%2C46833%2C1733267673070.meta.1733267675470.meta, worker=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,584 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T23:14:40,584 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5f9d64419733%2C46833%2C1733267673070.meta.1733267675470.meta, worker=5f9d64419733,41167,1733267673270 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T23:14:40,586 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d9981bd2d541debb970a37ac121e7ed, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,586 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f71f90ef2d2f59e885f2ad6ca1a99596, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,586 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e35e57b6f6cbff01d3e782fbf6efd1f, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,586 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=55eff67791415c655cef877182b9bbe1, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,586 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=212a9963c9cddede26a4eb521cff5254, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,586 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f514b27fbd5c6ef451e11852713ea906, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,586 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=73cd6e72f89a0d14d1d979f76c9c07a4, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,586 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=256869bf1dc8aa33be79adb19e13d134, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,587 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2db7cef12ba09a51b0a7204fc81deeec, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,587 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=384b2e2791a8bf699cb7996e1aa7469c, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,587 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5ddac1aa9a6f0af338b83490d6d952c2, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,587 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1d026a7a0318e7deb79fa16b1262ae2c, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,587 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4e6706440af7674db97bb48a05858115, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,587 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f00ca5d778884039f3248393a629dcfc, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,587 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5e6925efcdb64a8de08b4c4bbf3a3d2e, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,587 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=742c43a49ea3b763b8232c8e8c8966b3, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,587 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=10beaf4047510d958e4a559194242a57, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,587 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=89a698454f10c9fb6cfc108c35d8415a, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,587 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b7da3736c6b257b43fe66ea13807bc6, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,587 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a6de975926600fad463e13238b2ac8a9, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,587 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e5e4ecc042f874ec5567f7e86548256f, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,587 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23a123891f32f583441cd7569a6de1f2, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,588 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0f09ea7cbe3ceec5232d11c21a512a4, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,588 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e96b12de28d6c3047803f54a3b027459, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,588 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=553e1b6e96399dd43d3b8f7cfad54903, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,588 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=db9924063978f5acb21b073c5d94a7ad, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,588 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-03T23:14:40,588 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733267740588 2024-12-03T23:14:40,588 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733267800588 2024-12-03T23:14:40,588 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 6 msec 2024-12-03T23:14:40,588 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-03T23:14:40,588 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T23:14:40,588 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-03T23:14:40,588 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-03T23:14:40,588 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-03T23:14:40,588 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-03T23:14:40,588 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-03T23:14:40,589 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-03T23:14:40,589 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-03T23:14:40,590 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-03T23:14:40,590 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-03T23:14:40,590 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-03T23:14:40,590 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-03T23:14:40,591 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-03T23:14:40,591 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 256869bf1dc8aa33be79adb19e13d134, server=5f9d64419733,41167,1733267673270:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5f9d64419733,46833,1733267673070, splitWal=true, meta=true 2024-12-03T23:14:40,591 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5f9d64419733%2C46833%2C1733267673070.meta.1733267675470.meta, worker=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,591 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5f9d64419733%2C46833%2C1733267673070.meta.1733267675470.meta, worker=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,591 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 256869bf1dc8aa33be79adb19e13d134, server=5f9d64419733,41167,1733267673270:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5f9d64419733,46833,1733267673070, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,591 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5f9d64419733%2C46833%2C1733267673070.meta.1733267675470.meta, worker=5f9d64419733,41167,1733267673270 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,591 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5f9d64419733%2C46833%2C1733267673070.meta.1733267675470.meta, worker=5f9d64419733,41167,1733267673270 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,591 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-12-03T23:14:40,592 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=61 2024-12-03T23:14:40,592 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=5f9d64419733,41167,1733267673270, table=hbase:meta, region=1588230740 2024-12-03T23:14:40,592 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 5f9d64419733,41167,1733267673270, state=OPEN 2024-12-03T23:14:40,594 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44899-0x1017a9bb4a60002, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T23:14:40,594 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41167-0x1017a9bb4a60003, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T23:14:40,594 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44117-0x1017a9bb4a60000, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T23:14:40,594 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T23:14:40,594 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T23:14:40,594 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T23:14:40,595 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733267710595 2024-12-03T23:14:40,595 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-03T23:14:40,595 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 256869bf1dc8aa33be79adb19e13d134, server=5f9d64419733,41167,1733267673270:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5f9d64419733,46833,1733267673070, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T23:14:40,596 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-03T23:14:40,597 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-03T23:14:40,597 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5f9d64419733%2C46833%2C1733267673070.meta.1733267675470.meta, worker=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,597 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T23:14:40,597 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5f9d64419733%2C46833%2C1733267673070.meta.1733267675470.meta, worker=5f9d64419733,41167,1733267673270 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T23:14:40,600 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d9981bd2d541debb970a37ac121e7ed, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,600 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f71f90ef2d2f59e885f2ad6ca1a99596, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,600 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e35e57b6f6cbff01d3e782fbf6efd1f, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,600 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=55eff67791415c655cef877182b9bbe1, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,600 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=212a9963c9cddede26a4eb521cff5254, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,600 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f514b27fbd5c6ef451e11852713ea906, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,600 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=73cd6e72f89a0d14d1d979f76c9c07a4, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,601 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=256869bf1dc8aa33be79adb19e13d134, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,601 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2db7cef12ba09a51b0a7204fc81deeec, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,601 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=384b2e2791a8bf699cb7996e1aa7469c, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,601 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5ddac1aa9a6f0af338b83490d6d952c2, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,601 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1d026a7a0318e7deb79fa16b1262ae2c, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,601 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4e6706440af7674db97bb48a05858115, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,601 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f00ca5d778884039f3248393a629dcfc, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,601 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5e6925efcdb64a8de08b4c4bbf3a3d2e, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,601 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=742c43a49ea3b763b8232c8e8c8966b3, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,602 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=10beaf4047510d958e4a559194242a57, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,602 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=89a698454f10c9fb6cfc108c35d8415a, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,602 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b7da3736c6b257b43fe66ea13807bc6, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,602 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a6de975926600fad463e13238b2ac8a9, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,602 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e5e4ecc042f874ec5567f7e86548256f, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,602 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23a123891f32f583441cd7569a6de1f2, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,602 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0f09ea7cbe3ceec5232d11c21a512a4, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,602 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e96b12de28d6c3047803f54a3b027459, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,602 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=553e1b6e96399dd43d3b8f7cfad54903, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,603 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=db9924063978f5acb21b073c5d94a7ad, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,603 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-03T23:14:40,603 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733267740603 2024-12-03T23:14:40,603 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733267800603 2024-12-03T23:14:40,603 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-12-03T23:14:40,603 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-03T23:14:40,603 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T23:14:40,603 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-03T23:14:40,603 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-03T23:14:40,603 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-03T23:14:40,603 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-03T23:14:40,603 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-03T23:14:40,603 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-03T23:14:40,604 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-03T23:14:40,606 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-03T23:14:40,606 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-03T23:14:40,606 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-03T23:14:40,607 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-03T23:14:40,607 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 256869bf1dc8aa33be79adb19e13d134, server=5f9d64419733,41167,1733267673270:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5f9d64419733,46833,1733267673070, splitWal=true, meta=true 2024-12-03T23:14:40,608 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5f9d64419733%2C46833%2C1733267673070.meta.1733267675470.meta, worker=5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,608 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 256869bf1dc8aa33be79adb19e13d134, server=5f9d64419733,41167,1733267673270:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5f9d64419733,46833,1733267673070, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,608 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5f9d64419733%2C46833%2C1733267673070.meta.1733267675470.meta, worker=5f9d64419733,41167,1733267673270 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T23:14:40,608 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-12-03T23:14:40,609 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=61 2024-12-03T23:14:40,609 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=5f9d64419733,41167,1733267673270, table=hbase:meta, region=1588230740 2024-12-03T23:14:40,609 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 5f9d64419733,41167,1733267673270, state=OPEN 2024-12-03T23:14:40,611 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41167-0x1017a9bb4a60003, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T23:14:40,611 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44117-0x1017a9bb4a60000, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T23:14:40,611 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44899-0x1017a9bb4a60002, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T23:14:40,611 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T23:14:40,611 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T23:14:40,611 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T23:14:40,612 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733267710612 2024-12-03T23:14:40,612 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-03T23:14:40,612 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 256869bf1dc8aa33be79adb19e13d134, server=5f9d64419733,41167,1733267673270:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5f9d64419733,46833,1733267673070, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T23:14:40,614 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-03T23:14:40,617 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-03T23:14:40,617 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 256869bf1dc8aa33be79adb19e13d134, server=5f9d64419733,41167,1733267673270:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5f9d64419733,46833,1733267673070, splitWal=true, meta=true 2024-12-03T23:14:40,617 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T23:14:40,617 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 256869bf1dc8aa33be79adb19e13d134, server=5f9d64419733,41167,1733267673270:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5f9d64419733,46833,1733267673070, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1604) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T23:14:40,618 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d9981bd2d541debb970a37ac121e7ed, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,618 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f71f90ef2d2f59e885f2ad6ca1a99596, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,618 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e35e57b6f6cbff01d3e782fbf6efd1f, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,618 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=55eff67791415c655cef877182b9bbe1, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,618 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=212a9963c9cddede26a4eb521cff5254, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,618 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f514b27fbd5c6ef451e11852713ea906, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,618 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=73cd6e72f89a0d14d1d979f76c9c07a4, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,618 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=256869bf1dc8aa33be79adb19e13d134, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,619 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2db7cef12ba09a51b0a7204fc81deeec, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,619 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=384b2e2791a8bf699cb7996e1aa7469c, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,619 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5ddac1aa9a6f0af338b83490d6d952c2, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,619 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1d026a7a0318e7deb79fa16b1262ae2c, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,619 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4e6706440af7674db97bb48a05858115, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,619 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f00ca5d778884039f3248393a629dcfc, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,619 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5e6925efcdb64a8de08b4c4bbf3a3d2e, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,619 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=742c43a49ea3b763b8232c8e8c8966b3, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,619 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=10beaf4047510d958e4a559194242a57, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,620 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=89a698454f10c9fb6cfc108c35d8415a, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,620 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b7da3736c6b257b43fe66ea13807bc6, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,620 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a6de975926600fad463e13238b2ac8a9, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,620 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e5e4ecc042f874ec5567f7e86548256f, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,620 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23a123891f32f583441cd7569a6de1f2, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,620 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0f09ea7cbe3ceec5232d11c21a512a4, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,621 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e96b12de28d6c3047803f54a3b027459, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,621 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=553e1b6e96399dd43d3b8f7cfad54903, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,621 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=db9924063978f5acb21b073c5d94a7ad, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,621 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-03T23:14:40,621 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733267740621 2024-12-03T23:14:40,621 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733267800621 2024-12-03T23:14:40,621 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-12-03T23:14:40,621 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-03T23:14:40,621 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T23:14:40,621 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-03T23:14:40,621 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-03T23:14:40,622 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-03T23:14:40,622 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-03T23:14:40,622 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-03T23:14:40,622 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-03T23:14:40,623 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-03T23:14:40,625 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-03T23:14:40,625 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-03T23:14:40,626 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-03T23:14:40,626 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 256869bf1dc8aa33be79adb19e13d134, server=5f9d64419733,41167,1733267673270:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5f9d64419733,46833,1733267673070, splitWal=true, meta=true 2024-12-03T23:14:40,626 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 256869bf1dc8aa33be79adb19e13d134, server=5f9d64419733,41167,1733267673270:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5f9d64419733,46833,1733267673070, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-03T23:14:40,626 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-12-03T23:14:40,627 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=61 2024-12-03T23:14:40,627 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=5f9d64419733,41167,1733267673270, table=hbase:meta, region=1588230740 2024-12-03T23:14:40,628 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 5f9d64419733,41167,1733267673270, state=OPEN 2024-12-03T23:14:40,631 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44899-0x1017a9bb4a60002, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T23:14:40,631 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41167-0x1017a9bb4a60003, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T23:14:40,631 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44117-0x1017a9bb4a60000, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T23:14:40,631 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T23:14:40,631 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T23:14:40,631 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T23:14:40,632 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733267710632 2024-12-03T23:14:40,632 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-03T23:14:40,632 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 256869bf1dc8aa33be79adb19e13d134, server=5f9d64419733,41167,1733267673270:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5f9d64419733,46833,1733267673070, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T23:14:40,633 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-03T23:14:40,635 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=57, state=ROLLEDBACK, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 256869bf1dc8aa33be79adb19e13d134, server=5f9d64419733,41167,1733267673270:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5f9d64419733,46833,1733267673070, splitWal=true, meta=true exec-time=2.0960 sec 2024-12-03T23:14:40,637 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d9981bd2d541debb970a37ac121e7ed, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,637 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f71f90ef2d2f59e885f2ad6ca1a99596, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,637 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e35e57b6f6cbff01d3e782fbf6efd1f, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,637 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=55eff67791415c655cef877182b9bbe1, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,638 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=212a9963c9cddede26a4eb521cff5254, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,638 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f514b27fbd5c6ef451e11852713ea906, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,638 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=73cd6e72f89a0d14d1d979f76c9c07a4, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,638 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=256869bf1dc8aa33be79adb19e13d134, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,638 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2db7cef12ba09a51b0a7204fc81deeec, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,638 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=384b2e2791a8bf699cb7996e1aa7469c, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,638 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5ddac1aa9a6f0af338b83490d6d952c2, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,638 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1d026a7a0318e7deb79fa16b1262ae2c, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,638 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4e6706440af7674db97bb48a05858115, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,638 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f00ca5d778884039f3248393a629dcfc, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,639 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5e6925efcdb64a8de08b4c4bbf3a3d2e, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,639 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=742c43a49ea3b763b8232c8e8c8966b3, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,639 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=10beaf4047510d958e4a559194242a57, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,639 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=89a698454f10c9fb6cfc108c35d8415a, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,639 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b7da3736c6b257b43fe66ea13807bc6, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,639 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a6de975926600fad463e13238b2ac8a9, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,639 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e5e4ecc042f874ec5567f7e86548256f, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,639 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23a123891f32f583441cd7569a6de1f2, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,639 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0f09ea7cbe3ceec5232d11c21a512a4, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,639 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e96b12de28d6c3047803f54a3b027459, regionState=OPEN, lastHost=5f9d64419733,41167,1733267673270, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,640 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=553e1b6e96399dd43d3b8f7cfad54903, regionState=OPENING, lastHost=5f9d64419733,46833,1733267673070, regionLocation=5f9d64419733,41167,1733267673270, openSeqNum=2 2024-12-03T23:14:40,640 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=db9924063978f5acb21b073c5d94a7ad, regionState=OPEN, lastHost=5f9d64419733,44899,1733267673202, regionLocation=5f9d64419733,44899,1733267673202, openSeqNum=2 2024-12-03T23:14:40,640 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-03T23:14:40,640 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733267740640 2024-12-03T23:14:40,640 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733267800640 2024-12-03T23:14:40,640 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-12-03T23:14:40,660 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: master.assignment.TestRollbackSCP#testFailAndRollback Thread=345 (was 378), OpenFileDescriptor=628 (was 649), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=558 (was 558), ProcessCount=10 (was 11), AvailableMemoryMB=7705 (was 7755) 2024-12-03T23:14:40,660 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-03T23:14:40,660 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-03T23:14:40,660 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.tearDownAfterClass(TestRollbackSCP.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T23:14:40,660 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T23:14:40,661 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T23:14:40,661 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T23:14:40,661 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-03T23:14:40,661 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1801512384, stopped=false 2024-12-03T23:14:40,662 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=5f9d64419733,44117,1733267672173 2024-12-03T23:14:40,664 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41167-0x1017a9bb4a60003, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-03T23:14:40,664 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44117-0x1017a9bb4a60000, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-03T23:14:40,664 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44899-0x1017a9bb4a60002, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-03T23:14:40,664 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44117-0x1017a9bb4a60000, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T23:14:40,664 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44899-0x1017a9bb4a60002, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T23:14:40,664 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41167-0x1017a9bb4a60003, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T23:14:40,664 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T23:14:40,665 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-03T23:14:40,665 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:44899-0x1017a9bb4a60002, quorum=127.0.0.1:62825, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-03T23:14:40,665 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:41167-0x1017a9bb4a60003, quorum=127.0.0.1:62825, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-03T23:14:40,665 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.tearDownAfterClass(TestRollbackSCP.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T23:14:40,665 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T23:14:40,665 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:44117-0x1017a9bb4a60000, quorum=127.0.0.1:62825, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-03T23:14:40,665 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '5f9d64419733,44899,1733267673202' ***** 2024-12-03T23:14:40,665 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-03T23:14:40,665 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '5f9d64419733,41167,1733267673270' ***** 2024-12-03T23:14:40,665 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-03T23:14:40,666 INFO [RS:2;5f9d64419733:41167 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-03T23:14:40,666 INFO [RS:2;5f9d64419733:41167 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-03T23:14:40,666 INFO [RS:2;5f9d64419733:41167 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-03T23:14:40,666 INFO [RS:2;5f9d64419733:41167 {}] regionserver.HRegionServer(3091): Received CLOSE for 55eff67791415c655cef877182b9bbe1 2024-12-03T23:14:40,666 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-03T23:14:40,666 INFO [RS:1;5f9d64419733:44899 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-03T23:14:40,667 INFO [RS:1;5f9d64419733:44899 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-03T23:14:40,667 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-03T23:14:40,667 INFO [RS:1;5f9d64419733:44899 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-03T23:14:40,667 INFO [RS:1;5f9d64419733:44899 {}] regionserver.HRegionServer(3091): Received CLOSE for f00ca5d778884039f3248393a629dcfc 2024-12-03T23:14:40,667 INFO [RS:2;5f9d64419733:41167 {}] regionserver.HRegionServer(3091): Received CLOSE for 256869bf1dc8aa33be79adb19e13d134 2024-12-03T23:14:40,667 INFO [RS:2;5f9d64419733:41167 {}] regionserver.HRegionServer(3091): Received CLOSE for 5e6925efcdb64a8de08b4c4bbf3a3d2e 2024-12-03T23:14:40,667 INFO [RS:2;5f9d64419733:41167 {}] regionserver.HRegionServer(3091): Received CLOSE for 1d026a7a0318e7deb79fa16b1262ae2c 2024-12-03T23:14:40,667 INFO [RS:2;5f9d64419733:41167 {}] regionserver.HRegionServer(3091): Received CLOSE for 89a698454f10c9fb6cfc108c35d8415a 2024-12-03T23:14:40,667 INFO [RS:2;5f9d64419733:41167 {}] regionserver.HRegionServer(3091): Received CLOSE for e5e4ecc042f874ec5567f7e86548256f 2024-12-03T23:14:40,667 INFO [RS:2;5f9d64419733:41167 {}] regionserver.HRegionServer(3091): Received CLOSE for 553e1b6e96399dd43d3b8f7cfad54903 2024-12-03T23:14:40,667 INFO [RS:2;5f9d64419733:41167 {}] regionserver.HRegionServer(3091): Received CLOSE for e96b12de28d6c3047803f54a3b027459 2024-12-03T23:14:40,667 INFO [RS:2;5f9d64419733:41167 {}] regionserver.HRegionServer(3091): Received CLOSE for 2db7cef12ba09a51b0a7204fc81deeec 2024-12-03T23:14:40,667 INFO [RS:1;5f9d64419733:44899 {}] regionserver.HRegionServer(3091): Received CLOSE for 0e35e57b6f6cbff01d3e782fbf6efd1f 2024-12-03T23:14:40,667 INFO [RS:2;5f9d64419733:41167 {}] regionserver.HRegionServer(3091): Received CLOSE for 2b7da3736c6b257b43fe66ea13807bc6 2024-12-03T23:14:40,667 INFO [RS:1;5f9d64419733:44899 {}] regionserver.HRegionServer(3091): Received CLOSE for a0f09ea7cbe3ceec5232d11c21a512a4 2024-12-03T23:14:40,667 INFO [RS:2;5f9d64419733:41167 {}] regionserver.HRegionServer(3091): Received CLOSE for 4e6706440af7674db97bb48a05858115 2024-12-03T23:14:40,667 INFO [RS:1;5f9d64419733:44899 {}] regionserver.HRegionServer(3091): Received CLOSE for f514b27fbd5c6ef451e11852713ea906 2024-12-03T23:14:40,667 INFO [RS:2;5f9d64419733:41167 {}] regionserver.HRegionServer(3091): Received CLOSE for 212a9963c9cddede26a4eb521cff5254 2024-12-03T23:14:40,667 INFO [RS:1;5f9d64419733:44899 {}] regionserver.HRegionServer(3091): Received CLOSE for db9924063978f5acb21b073c5d94a7ad 2024-12-03T23:14:40,667 INFO [RS:2;5f9d64419733:41167 {}] regionserver.HRegionServer(3091): Received CLOSE for f71f90ef2d2f59e885f2ad6ca1a99596 2024-12-03T23:14:40,667 INFO [RS:1;5f9d64419733:44899 {}] regionserver.HRegionServer(3091): Received CLOSE for 5ddac1aa9a6f0af338b83490d6d952c2 2024-12-03T23:14:40,667 INFO [RS:1;5f9d64419733:44899 {}] regionserver.HRegionServer(3091): Received CLOSE for 10beaf4047510d958e4a559194242a57 2024-12-03T23:14:40,667 INFO [RS:1;5f9d64419733:44899 {}] regionserver.HRegionServer(3091): Received CLOSE for 73cd6e72f89a0d14d1d979f76c9c07a4 2024-12-03T23:14:40,667 INFO [RS:2;5f9d64419733:41167 {}] regionserver.HRegionServer(959): stopping server 5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,667 INFO [RS:1;5f9d64419733:44899 {}] regionserver.HRegionServer(3091): Received CLOSE for a6de975926600fad463e13238b2ac8a9 2024-12-03T23:14:40,667 INFO [RS:2;5f9d64419733:41167 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-03T23:14:40,667 INFO [RS:1;5f9d64419733:44899 {}] regionserver.HRegionServer(959): stopping server 5f9d64419733,44899,1733267673202 2024-12-03T23:14:40,667 INFO [RS:1;5f9d64419733:44899 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-03T23:14:40,668 INFO [RS:2;5f9d64419733:41167 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;5f9d64419733:41167. 2024-12-03T23:14:40,668 INFO [RS:1;5f9d64419733:44899 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;5f9d64419733:44899. 2024-12-03T23:14:40,668 DEBUG [RS:1;5f9d64419733:44899 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T23:14:40,668 DEBUG [RS:2;5f9d64419733:41167 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T23:14:40,668 DEBUG [RS:1;5f9d64419733:44899 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T23:14:40,668 DEBUG [RS:2;5f9d64419733:41167 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T23:14:40,668 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing f00ca5d778884039f3248393a629dcfc, disabling compactions & flushes 2024-12-03T23:14:40,668 INFO [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,nnn,1733267676090.f00ca5d778884039f3248393a629dcfc. 2024-12-03T23:14:40,668 INFO [RS:1;5f9d64419733:44899 {}] regionserver.HRegionServer(1321): Waiting on 9 regions to close 2024-12-03T23:14:40,668 INFO [RS:2;5f9d64419733:41167 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-03T23:14:40,668 INFO [RS:2;5f9d64419733:41167 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-03T23:14:40,668 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,nnn,1733267676090.f00ca5d778884039f3248393a629dcfc. 2024-12-03T23:14:40,668 DEBUG [RS:1;5f9d64419733:44899 {}] regionserver.HRegionServer(1325): Online Regions={f00ca5d778884039f3248393a629dcfc=test,nnn,1733267676090.f00ca5d778884039f3248393a629dcfc., 0e35e57b6f6cbff01d3e782fbf6efd1f=test,ccc,1733267676090.0e35e57b6f6cbff01d3e782fbf6efd1f., a0f09ea7cbe3ceec5232d11c21a512a4=test,www,1733267676090.a0f09ea7cbe3ceec5232d11c21a512a4., f514b27fbd5c6ef451e11852713ea906=test,fff,1733267676090.f514b27fbd5c6ef451e11852713ea906., db9924063978f5acb21b073c5d94a7ad=test,zzz,1733267676090.db9924063978f5acb21b073c5d94a7ad., 5ddac1aa9a6f0af338b83490d6d952c2=test,kkk,1733267676090.5ddac1aa9a6f0af338b83490d6d952c2., 10beaf4047510d958e4a559194242a57=test,qqq,1733267676090.10beaf4047510d958e4a559194242a57., 73cd6e72f89a0d14d1d979f76c9c07a4=test,ggg,1733267676090.73cd6e72f89a0d14d1d979f76c9c07a4., a6de975926600fad463e13238b2ac8a9=test,ttt,1733267676090.a6de975926600fad463e13238b2ac8a9.} 2024-12-03T23:14:40,668 INFO [RS:2;5f9d64419733:41167 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-03T23:14:40,668 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,nnn,1733267676090.f00ca5d778884039f3248393a629dcfc. after waiting 0 ms 2024-12-03T23:14:40,668 INFO [RS:2;5f9d64419733:41167 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-03T23:14:40,668 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,nnn,1733267676090.f00ca5d778884039f3248393a629dcfc. 2024-12-03T23:14:40,668 DEBUG [RS:1;5f9d64419733:44899 {}] regionserver.HRegionServer(1351): Waiting on 0e35e57b6f6cbff01d3e782fbf6efd1f, 10beaf4047510d958e4a559194242a57, 384b2e2791a8bf699cb7996e1aa7469c, 5ddac1aa9a6f0af338b83490d6d952c2, 73cd6e72f89a0d14d1d979f76c9c07a4, a0f09ea7cbe3ceec5232d11c21a512a4, a6de975926600fad463e13238b2ac8a9, db9924063978f5acb21b073c5d94a7ad, f00ca5d778884039f3248393a629dcfc, f514b27fbd5c6ef451e11852713ea906 2024-12-03T23:14:40,669 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 55eff67791415c655cef877182b9bbe1, disabling compactions & flushes 2024-12-03T23:14:40,669 INFO [regionserver/5f9d64419733:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-03T23:14:40,669 INFO [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ddd,1733267676090.55eff67791415c655cef877182b9bbe1. 2024-12-03T23:14:40,669 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ddd,1733267676090.55eff67791415c655cef877182b9bbe1. 2024-12-03T23:14:40,669 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ddd,1733267676090.55eff67791415c655cef877182b9bbe1. after waiting 0 ms 2024-12-03T23:14:40,669 INFO [RS:2;5f9d64419733:41167 {}] regionserver.HRegionServer(1321): Waiting on 14 regions to close 2024-12-03T23:14:40,669 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ddd,1733267676090.55eff67791415c655cef877182b9bbe1. 2024-12-03T23:14:40,669 DEBUG [RS:2;5f9d64419733:41167 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 1d026a7a0318e7deb79fa16b1262ae2c, 212a9963c9cddede26a4eb521cff5254, 256869bf1dc8aa33be79adb19e13d134, 2b7da3736c6b257b43fe66ea13807bc6, 2db7cef12ba09a51b0a7204fc81deeec, 4e6706440af7674db97bb48a05858115, 553e1b6e96399dd43d3b8f7cfad54903, 55eff67791415c655cef877182b9bbe1, 5e6925efcdb64a8de08b4c4bbf3a3d2e, 89a698454f10c9fb6cfc108c35d8415a, e5e4ecc042f874ec5567f7e86548256f, e96b12de28d6c3047803f54a3b027459, f71f90ef2d2f59e885f2ad6ca1a99596 2024-12-03T23:14:40,669 DEBUG [RS_CLOSE_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-03T23:14:40,669 INFO [RS_CLOSE_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-03T23:14:40,669 DEBUG [RS_CLOSE_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-03T23:14:40,669 DEBUG [RS_CLOSE_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-03T23:14:40,669 DEBUG [RS_CLOSE_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-03T23:14:40,669 INFO [regionserver/5f9d64419733:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-03T23:14:40,669 INFO [RS_CLOSE_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=2.64 KB heapSize=5.42 KB 2024-12-03T23:14:40,674 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/default/test/55eff67791415c655cef877182b9bbe1/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-03T23:14:40,674 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/default/test/f00ca5d778884039f3248393a629dcfc/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-03T23:14:40,678 INFO [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,nnn,1733267676090.f00ca5d778884039f3248393a629dcfc. 2024-12-03T23:14:40,678 INFO [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ddd,1733267676090.55eff67791415c655cef877182b9bbe1. 2024-12-03T23:14:40,678 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 55eff67791415c655cef877182b9bbe1: Waiting for close lock at 1733267680669Running coprocessor pre-close hooks at 1733267680669Disabling compacts and flushes for region at 1733267680669Disabling writes for close at 1733267680669Writing region close event to WAL at 1733267680669Running coprocessor post-close hooks at 1733267680675 (+6 ms)Closed at 1733267680678 (+3 ms) 2024-12-03T23:14:40,678 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for f00ca5d778884039f3248393a629dcfc: Waiting for close lock at 1733267680667Running coprocessor pre-close hooks at 1733267680667Disabling compacts and flushes for region at 1733267680668 (+1 ms)Disabling writes for close at 1733267680668Writing region close event to WAL at 1733267680669 (+1 ms)Running coprocessor post-close hooks at 1733267680675 (+6 ms)Closed at 1733267680678 (+3 ms) 2024-12-03T23:14:40,678 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,nnn,1733267676090.f00ca5d778884039f3248393a629dcfc. 2024-12-03T23:14:40,678 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ddd,1733267676090.55eff67791415c655cef877182b9bbe1. 2024-12-03T23:14:40,679 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 0e35e57b6f6cbff01d3e782fbf6efd1f, disabling compactions & flushes 2024-12-03T23:14:40,679 INFO [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ccc,1733267676090.0e35e57b6f6cbff01d3e782fbf6efd1f. 2024-12-03T23:14:40,679 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 256869bf1dc8aa33be79adb19e13d134, disabling compactions & flushes 2024-12-03T23:14:40,679 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ccc,1733267676090.0e35e57b6f6cbff01d3e782fbf6efd1f. 2024-12-03T23:14:40,679 INFO [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,hhh,1733267676090.256869bf1dc8aa33be79adb19e13d134. 2024-12-03T23:14:40,679 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ccc,1733267676090.0e35e57b6f6cbff01d3e782fbf6efd1f. after waiting 0 ms 2024-12-03T23:14:40,679 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ccc,1733267676090.0e35e57b6f6cbff01d3e782fbf6efd1f. 2024-12-03T23:14:40,679 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,hhh,1733267676090.256869bf1dc8aa33be79adb19e13d134. 2024-12-03T23:14:40,679 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,hhh,1733267676090.256869bf1dc8aa33be79adb19e13d134. after waiting 0 ms 2024-12-03T23:14:40,679 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,hhh,1733267676090.256869bf1dc8aa33be79adb19e13d134. 2024-12-03T23:14:40,684 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/default/test/0e35e57b6f6cbff01d3e782fbf6efd1f/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-03T23:14:40,685 INFO [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ccc,1733267676090.0e35e57b6f6cbff01d3e782fbf6efd1f. 2024-12-03T23:14:40,685 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 0e35e57b6f6cbff01d3e782fbf6efd1f: Waiting for close lock at 1733267680679Running coprocessor pre-close hooks at 1733267680679Disabling compacts and flushes for region at 1733267680679Disabling writes for close at 1733267680679Writing region close event to WAL at 1733267680679Running coprocessor post-close hooks at 1733267680684 (+5 ms)Closed at 1733267680685 (+1 ms) 2024-12-03T23:14:40,685 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ccc,1733267676090.0e35e57b6f6cbff01d3e782fbf6efd1f. 2024-12-03T23:14:40,685 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing a0f09ea7cbe3ceec5232d11c21a512a4, disabling compactions & flushes 2024-12-03T23:14:40,685 INFO [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,www,1733267676090.a0f09ea7cbe3ceec5232d11c21a512a4. 2024-12-03T23:14:40,685 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,www,1733267676090.a0f09ea7cbe3ceec5232d11c21a512a4. 2024-12-03T23:14:40,685 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,www,1733267676090.a0f09ea7cbe3ceec5232d11c21a512a4. after waiting 0 ms 2024-12-03T23:14:40,685 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,www,1733267676090.a0f09ea7cbe3ceec5232d11c21a512a4. 2024-12-03T23:14:40,685 DEBUG [RS_CLOSE_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/hbase/meta/1588230740/.tmp/info/679ca6bce73d4c73a0c12e0fd25b2a62 is 138, key is test,hhh,1733267676090.256869bf1dc8aa33be79adb19e13d134./info:regioninfo/1733267679882/Put/seqid=0 2024-12-03T23:14:40,686 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/default/test/256869bf1dc8aa33be79adb19e13d134/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-03T23:14:40,688 INFO [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,hhh,1733267676090.256869bf1dc8aa33be79adb19e13d134. 2024-12-03T23:14:40,688 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 256869bf1dc8aa33be79adb19e13d134: Waiting for close lock at 1733267680679Running coprocessor pre-close hooks at 1733267680679Disabling compacts and flushes for region at 1733267680679Disabling writes for close at 1733267680679Writing region close event to WAL at 1733267680680 (+1 ms)Running coprocessor post-close hooks at 1733267680688 (+8 ms)Closed at 1733267680688 2024-12-03T23:14:40,688 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,hhh,1733267676090.256869bf1dc8aa33be79adb19e13d134. 2024-12-03T23:14:40,688 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 5e6925efcdb64a8de08b4c4bbf3a3d2e, disabling compactions & flushes 2024-12-03T23:14:40,688 INFO [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ooo,1733267676090.5e6925efcdb64a8de08b4c4bbf3a3d2e. 2024-12-03T23:14:40,688 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ooo,1733267676090.5e6925efcdb64a8de08b4c4bbf3a3d2e. 2024-12-03T23:14:40,688 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ooo,1733267676090.5e6925efcdb64a8de08b4c4bbf3a3d2e. after waiting 0 ms 2024-12-03T23:14:40,688 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ooo,1733267676090.5e6925efcdb64a8de08b4c4bbf3a3d2e. 2024-12-03T23:14:40,690 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/default/test/a0f09ea7cbe3ceec5232d11c21a512a4/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-03T23:14:40,691 INFO [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,www,1733267676090.a0f09ea7cbe3ceec5232d11c21a512a4. 2024-12-03T23:14:40,691 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for a0f09ea7cbe3ceec5232d11c21a512a4: Waiting for close lock at 1733267680685Running coprocessor pre-close hooks at 1733267680685Disabling compacts and flushes for region at 1733267680685Disabling writes for close at 1733267680685Writing region close event to WAL at 1733267680686 (+1 ms)Running coprocessor post-close hooks at 1733267680691 (+5 ms)Closed at 1733267680691 2024-12-03T23:14:40,691 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,www,1733267676090.a0f09ea7cbe3ceec5232d11c21a512a4. 2024-12-03T23:14:40,691 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing f514b27fbd5c6ef451e11852713ea906, disabling compactions & flushes 2024-12-03T23:14:40,691 INFO [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,fff,1733267676090.f514b27fbd5c6ef451e11852713ea906. 2024-12-03T23:14:40,691 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,fff,1733267676090.f514b27fbd5c6ef451e11852713ea906. 2024-12-03T23:14:40,691 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,fff,1733267676090.f514b27fbd5c6ef451e11852713ea906. after waiting 0 ms 2024-12-03T23:14:40,692 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,fff,1733267676090.f514b27fbd5c6ef451e11852713ea906. 2024-12-03T23:14:40,701 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/default/test/5e6925efcdb64a8de08b4c4bbf3a3d2e/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-03T23:14:40,701 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/default/test/f514b27fbd5c6ef451e11852713ea906/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-03T23:14:40,702 INFO [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ooo,1733267676090.5e6925efcdb64a8de08b4c4bbf3a3d2e. 2024-12-03T23:14:40,702 INFO [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,fff,1733267676090.f514b27fbd5c6ef451e11852713ea906. 2024-12-03T23:14:40,702 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for f514b27fbd5c6ef451e11852713ea906: Waiting for close lock at 1733267680691Running coprocessor pre-close hooks at 1733267680691Disabling compacts and flushes for region at 1733267680691Disabling writes for close at 1733267680692 (+1 ms)Writing region close event to WAL at 1733267680693 (+1 ms)Running coprocessor post-close hooks at 1733267680702 (+9 ms)Closed at 1733267680702 2024-12-03T23:14:40,702 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 5e6925efcdb64a8de08b4c4bbf3a3d2e: Waiting for close lock at 1733267680688Running coprocessor pre-close hooks at 1733267680688Disabling compacts and flushes for region at 1733267680688Disabling writes for close at 1733267680688Writing region close event to WAL at 1733267680690 (+2 ms)Running coprocessor post-close hooks at 1733267680702 (+12 ms)Closed at 1733267680702 2024-12-03T23:14:40,702 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,fff,1733267676090.f514b27fbd5c6ef451e11852713ea906. 2024-12-03T23:14:40,702 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ooo,1733267676090.5e6925efcdb64a8de08b4c4bbf3a3d2e. 2024-12-03T23:14:40,703 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing db9924063978f5acb21b073c5d94a7ad, disabling compactions & flushes 2024-12-03T23:14:40,703 INFO [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,zzz,1733267676090.db9924063978f5acb21b073c5d94a7ad. 2024-12-03T23:14:40,703 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 1d026a7a0318e7deb79fa16b1262ae2c, disabling compactions & flushes 2024-12-03T23:14:40,703 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,zzz,1733267676090.db9924063978f5acb21b073c5d94a7ad. 2024-12-03T23:14:40,703 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,zzz,1733267676090.db9924063978f5acb21b073c5d94a7ad. after waiting 0 ms 2024-12-03T23:14:40,703 INFO [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,lll,1733267676090.1d026a7a0318e7deb79fa16b1262ae2c. 2024-12-03T23:14:40,703 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,zzz,1733267676090.db9924063978f5acb21b073c5d94a7ad. 2024-12-03T23:14:40,703 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,lll,1733267676090.1d026a7a0318e7deb79fa16b1262ae2c. 2024-12-03T23:14:40,703 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,lll,1733267676090.1d026a7a0318e7deb79fa16b1262ae2c. after waiting 0 ms 2024-12-03T23:14:40,703 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,lll,1733267676090.1d026a7a0318e7deb79fa16b1262ae2c. 2024-12-03T23:14:40,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38883 is added to blk_1073741869_1045 (size=8177) 2024-12-03T23:14:40,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45029 is added to blk_1073741869_1045 (size=8177) 2024-12-03T23:14:40,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44333 is added to blk_1073741869_1045 (size=8177) 2024-12-03T23:14:40,710 INFO [RS_CLOSE_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.64 KB at sequenceid=71 (bloomFilter=true), to=hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/hbase/meta/1588230740/.tmp/info/679ca6bce73d4c73a0c12e0fd25b2a62 2024-12-03T23:14:40,711 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/default/test/db9924063978f5acb21b073c5d94a7ad/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-03T23:14:40,711 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/default/test/1d026a7a0318e7deb79fa16b1262ae2c/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-03T23:14:40,713 INFO [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,lll,1733267676090.1d026a7a0318e7deb79fa16b1262ae2c. 2024-12-03T23:14:40,713 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 1d026a7a0318e7deb79fa16b1262ae2c: Waiting for close lock at 1733267680703Running coprocessor pre-close hooks at 1733267680703Disabling compacts and flushes for region at 1733267680703Disabling writes for close at 1733267680703Writing region close event to WAL at 1733267680705 (+2 ms)Running coprocessor post-close hooks at 1733267680713 (+8 ms)Closed at 1733267680713 2024-12-03T23:14:40,713 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,lll,1733267676090.1d026a7a0318e7deb79fa16b1262ae2c. 2024-12-03T23:14:40,713 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 89a698454f10c9fb6cfc108c35d8415a, disabling compactions & flushes 2024-12-03T23:14:40,713 INFO [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,rrr,1733267676090.89a698454f10c9fb6cfc108c35d8415a. 2024-12-03T23:14:40,714 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,rrr,1733267676090.89a698454f10c9fb6cfc108c35d8415a. 2024-12-03T23:14:40,714 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,rrr,1733267676090.89a698454f10c9fb6cfc108c35d8415a. after waiting 0 ms 2024-12-03T23:14:40,714 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,rrr,1733267676090.89a698454f10c9fb6cfc108c35d8415a. 2024-12-03T23:14:40,714 INFO [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,zzz,1733267676090.db9924063978f5acb21b073c5d94a7ad. 2024-12-03T23:14:40,714 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for db9924063978f5acb21b073c5d94a7ad: Waiting for close lock at 1733267680702Running coprocessor pre-close hooks at 1733267680702Disabling compacts and flushes for region at 1733267680703 (+1 ms)Disabling writes for close at 1733267680703Writing region close event to WAL at 1733267680705 (+2 ms)Running coprocessor post-close hooks at 1733267680714 (+9 ms)Closed at 1733267680714 2024-12-03T23:14:40,714 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,zzz,1733267676090.db9924063978f5acb21b073c5d94a7ad. 2024-12-03T23:14:40,714 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 5ddac1aa9a6f0af338b83490d6d952c2, disabling compactions & flushes 2024-12-03T23:14:40,714 INFO [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,kkk,1733267676090.5ddac1aa9a6f0af338b83490d6d952c2. 2024-12-03T23:14:40,714 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,kkk,1733267676090.5ddac1aa9a6f0af338b83490d6d952c2. 2024-12-03T23:14:40,714 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,kkk,1733267676090.5ddac1aa9a6f0af338b83490d6d952c2. after waiting 0 ms 2024-12-03T23:14:40,714 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,kkk,1733267676090.5ddac1aa9a6f0af338b83490d6d952c2. 2024-12-03T23:14:40,720 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/default/test/5ddac1aa9a6f0af338b83490d6d952c2/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-03T23:14:40,721 DEBUG [RS_CLOSE_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/hbase/meta/1588230740/.tmp/info/679ca6bce73d4c73a0c12e0fd25b2a62 as hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/hbase/meta/1588230740/info/679ca6bce73d4c73a0c12e0fd25b2a62 2024-12-03T23:14:40,721 INFO [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,kkk,1733267676090.5ddac1aa9a6f0af338b83490d6d952c2. 2024-12-03T23:14:40,721 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/default/test/89a698454f10c9fb6cfc108c35d8415a/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-03T23:14:40,722 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 5ddac1aa9a6f0af338b83490d6d952c2: Waiting for close lock at 1733267680714Running coprocessor pre-close hooks at 1733267680714Disabling compacts and flushes for region at 1733267680714Disabling writes for close at 1733267680714Writing region close event to WAL at 1733267680715 (+1 ms)Running coprocessor post-close hooks at 1733267680721 (+6 ms)Closed at 1733267680721 2024-12-03T23:14:40,722 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,kkk,1733267676090.5ddac1aa9a6f0af338b83490d6d952c2. 2024-12-03T23:14:40,722 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 10beaf4047510d958e4a559194242a57, disabling compactions & flushes 2024-12-03T23:14:40,722 INFO [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,qqq,1733267676090.10beaf4047510d958e4a559194242a57. 2024-12-03T23:14:40,722 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,qqq,1733267676090.10beaf4047510d958e4a559194242a57. 2024-12-03T23:14:40,722 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,qqq,1733267676090.10beaf4047510d958e4a559194242a57. after waiting 0 ms 2024-12-03T23:14:40,722 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,qqq,1733267676090.10beaf4047510d958e4a559194242a57. 2024-12-03T23:14:40,723 INFO [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,rrr,1733267676090.89a698454f10c9fb6cfc108c35d8415a. 2024-12-03T23:14:40,723 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 89a698454f10c9fb6cfc108c35d8415a: Waiting for close lock at 1733267680713Running coprocessor pre-close hooks at 1733267680713Disabling compacts and flushes for region at 1733267680713Disabling writes for close at 1733267680714 (+1 ms)Writing region close event to WAL at 1733267680716 (+2 ms)Running coprocessor post-close hooks at 1733267680722 (+6 ms)Closed at 1733267680722 2024-12-03T23:14:40,723 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,rrr,1733267676090.89a698454f10c9fb6cfc108c35d8415a. 2024-12-03T23:14:40,723 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing e5e4ecc042f874ec5567f7e86548256f, disabling compactions & flushes 2024-12-03T23:14:40,723 INFO [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,uuu,1733267676090.e5e4ecc042f874ec5567f7e86548256f. 2024-12-03T23:14:40,723 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,uuu,1733267676090.e5e4ecc042f874ec5567f7e86548256f. 2024-12-03T23:14:40,723 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,uuu,1733267676090.e5e4ecc042f874ec5567f7e86548256f. after waiting 0 ms 2024-12-03T23:14:40,723 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,uuu,1733267676090.e5e4ecc042f874ec5567f7e86548256f. 2024-12-03T23:14:40,728 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/default/test/10beaf4047510d958e4a559194242a57/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-03T23:14:40,729 INFO [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,qqq,1733267676090.10beaf4047510d958e4a559194242a57. 2024-12-03T23:14:40,729 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 10beaf4047510d958e4a559194242a57: Waiting for close lock at 1733267680722Running coprocessor pre-close hooks at 1733267680722Disabling compacts and flushes for region at 1733267680722Disabling writes for close at 1733267680722Writing region close event to WAL at 1733267680723 (+1 ms)Running coprocessor post-close hooks at 1733267680729 (+6 ms)Closed at 1733267680729 2024-12-03T23:14:40,729 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/default/test/e5e4ecc042f874ec5567f7e86548256f/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-03T23:14:40,730 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,qqq,1733267676090.10beaf4047510d958e4a559194242a57. 2024-12-03T23:14:40,730 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 73cd6e72f89a0d14d1d979f76c9c07a4, disabling compactions & flushes 2024-12-03T23:14:40,730 INFO [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ggg,1733267676090.73cd6e72f89a0d14d1d979f76c9c07a4. 2024-12-03T23:14:40,730 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ggg,1733267676090.73cd6e72f89a0d14d1d979f76c9c07a4. 2024-12-03T23:14:40,730 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ggg,1733267676090.73cd6e72f89a0d14d1d979f76c9c07a4. after waiting 0 ms 2024-12-03T23:14:40,730 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ggg,1733267676090.73cd6e72f89a0d14d1d979f76c9c07a4. 2024-12-03T23:14:40,730 INFO [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,uuu,1733267676090.e5e4ecc042f874ec5567f7e86548256f. 2024-12-03T23:14:40,730 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for e5e4ecc042f874ec5567f7e86548256f: Waiting for close lock at 1733267680723Running coprocessor pre-close hooks at 1733267680723Disabling compacts and flushes for region at 1733267680723Disabling writes for close at 1733267680723Writing region close event to WAL at 1733267680724 (+1 ms)Running coprocessor post-close hooks at 1733267680730 (+6 ms)Closed at 1733267680730 2024-12-03T23:14:40,731 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,uuu,1733267676090.e5e4ecc042f874ec5567f7e86548256f. 2024-12-03T23:14:40,731 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 553e1b6e96399dd43d3b8f7cfad54903, disabling compactions & flushes 2024-12-03T23:14:40,731 INFO [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,yyy,1733267676090.553e1b6e96399dd43d3b8f7cfad54903. 2024-12-03T23:14:40,731 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,yyy,1733267676090.553e1b6e96399dd43d3b8f7cfad54903. 2024-12-03T23:14:40,731 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,yyy,1733267676090.553e1b6e96399dd43d3b8f7cfad54903. after waiting 0 ms 2024-12-03T23:14:40,731 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,yyy,1733267676090.553e1b6e96399dd43d3b8f7cfad54903. 2024-12-03T23:14:40,732 INFO [RS_CLOSE_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/hbase/meta/1588230740/info/679ca6bce73d4c73a0c12e0fd25b2a62, entries=24, sequenceid=71, filesize=8.0 K 2024-12-03T23:14:40,734 INFO [RS_CLOSE_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~2.64 KB/2708, heapSize ~4.66 KB/4768, currentSize=0 B/0 for 1588230740 in 65ms, sequenceid=71, compaction requested=false 2024-12-03T23:14:40,736 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/default/test/73cd6e72f89a0d14d1d979f76c9c07a4/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-03T23:14:40,737 INFO [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ggg,1733267676090.73cd6e72f89a0d14d1d979f76c9c07a4. 2024-12-03T23:14:40,737 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 73cd6e72f89a0d14d1d979f76c9c07a4: Waiting for close lock at 1733267680730Running coprocessor pre-close hooks at 1733267680730Disabling compacts and flushes for region at 1733267680730Disabling writes for close at 1733267680730Writing region close event to WAL at 1733267680731 (+1 ms)Running coprocessor post-close hooks at 1733267680737 (+6 ms)Closed at 1733267680737 2024-12-03T23:14:40,737 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ggg,1733267676090.73cd6e72f89a0d14d1d979f76c9c07a4. 2024-12-03T23:14:40,737 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing a6de975926600fad463e13238b2ac8a9, disabling compactions & flushes 2024-12-03T23:14:40,737 INFO [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ttt,1733267676090.a6de975926600fad463e13238b2ac8a9. 2024-12-03T23:14:40,737 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ttt,1733267676090.a6de975926600fad463e13238b2ac8a9. 2024-12-03T23:14:40,737 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ttt,1733267676090.a6de975926600fad463e13238b2ac8a9. after waiting 0 ms 2024-12-03T23:14:40,737 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ttt,1733267676090.a6de975926600fad463e13238b2ac8a9. 2024-12-03T23:14:40,740 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/default/test/553e1b6e96399dd43d3b8f7cfad54903/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-03T23:14:40,740 INFO [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,yyy,1733267676090.553e1b6e96399dd43d3b8f7cfad54903. 2024-12-03T23:14:40,740 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 553e1b6e96399dd43d3b8f7cfad54903: Waiting for close lock at 1733267680731Running coprocessor pre-close hooks at 1733267680731Disabling compacts and flushes for region at 1733267680731Disabling writes for close at 1733267680731Writing region close event to WAL at 1733267680734 (+3 ms)Running coprocessor post-close hooks at 1733267680740 (+6 ms)Closed at 1733267680740 2024-12-03T23:14:40,741 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,yyy,1733267676090.553e1b6e96399dd43d3b8f7cfad54903. 2024-12-03T23:14:40,741 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing e96b12de28d6c3047803f54a3b027459, disabling compactions & flushes 2024-12-03T23:14:40,741 INFO [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,xxx,1733267676090.e96b12de28d6c3047803f54a3b027459. 2024-12-03T23:14:40,741 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,xxx,1733267676090.e96b12de28d6c3047803f54a3b027459. 2024-12-03T23:14:40,741 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,xxx,1733267676090.e96b12de28d6c3047803f54a3b027459. after waiting 0 ms 2024-12-03T23:14:40,741 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,xxx,1733267676090.e96b12de28d6c3047803f54a3b027459. 2024-12-03T23:14:40,742 DEBUG [RS_CLOSE_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/hbase/meta/1588230740/recovered.edits/74.seqid, newMaxSeqId=74, maxSeqId=60 2024-12-03T23:14:40,743 DEBUG [RS_CLOSE_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-03T23:14:40,743 INFO [RS_CLOSE_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-03T23:14:40,743 DEBUG [RS_CLOSE_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733267680669Running coprocessor pre-close hooks at 1733267680669Disabling compacts and flushes for region at 1733267680669Disabling writes for close at 1733267680669Obtaining lock to block concurrent updates at 1733267680669Preparing flush snapshotting stores in 1588230740 at 1733267680669Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=2708, getHeapSize=5488, getOffHeapSize=0, getCellsCount=24 at 1733267680676 (+7 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733267680679 (+3 ms)Flushing 1588230740/info: creating writer at 1733267680679Flushing 1588230740/info: appending metadata at 1733267680685 (+6 ms)Flushing 1588230740/info: closing flushed file at 1733267680685Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@15c362cb: reopening flushed file at 1733267680720 (+35 ms)Finished flush of dataSize ~2.64 KB/2708, heapSize ~4.66 KB/4768, currentSize=0 B/0 for 1588230740 in 65ms, sequenceid=71, compaction requested=false at 1733267680734 (+14 ms)Writing region close event to WAL at 1733267680737 (+3 ms)Running coprocessor post-close hooks at 1733267680742 (+5 ms)Closed at 1733267680743 (+1 ms) 2024-12-03T23:14:40,743 DEBUG [RS_CLOSE_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-03T23:14:40,744 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/default/test/a6de975926600fad463e13238b2ac8a9/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-03T23:14:40,745 INFO [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ttt,1733267676090.a6de975926600fad463e13238b2ac8a9. 2024-12-03T23:14:40,745 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for a6de975926600fad463e13238b2ac8a9: Waiting for close lock at 1733267680737Running coprocessor pre-close hooks at 1733267680737Disabling compacts and flushes for region at 1733267680737Disabling writes for close at 1733267680737Writing region close event to WAL at 1733267680738 (+1 ms)Running coprocessor post-close hooks at 1733267680745 (+7 ms)Closed at 1733267680745 2024-12-03T23:14:40,745 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ttt,1733267676090.a6de975926600fad463e13238b2ac8a9. 2024-12-03T23:14:40,745 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/default/test/e96b12de28d6c3047803f54a3b027459/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-03T23:14:40,746 INFO [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,xxx,1733267676090.e96b12de28d6c3047803f54a3b027459. 2024-12-03T23:14:40,746 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for e96b12de28d6c3047803f54a3b027459: Waiting for close lock at 1733267680741Running coprocessor pre-close hooks at 1733267680741Disabling compacts and flushes for region at 1733267680741Disabling writes for close at 1733267680741Writing region close event to WAL at 1733267680741Running coprocessor post-close hooks at 1733267680746 (+5 ms)Closed at 1733267680746 2024-12-03T23:14:40,746 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,xxx,1733267676090.e96b12de28d6c3047803f54a3b027459. 2024-12-03T23:14:40,746 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 2db7cef12ba09a51b0a7204fc81deeec, disabling compactions & flushes 2024-12-03T23:14:40,746 INFO [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,iii,1733267676090.2db7cef12ba09a51b0a7204fc81deeec. 2024-12-03T23:14:40,746 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,iii,1733267676090.2db7cef12ba09a51b0a7204fc81deeec. 2024-12-03T23:14:40,746 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,iii,1733267676090.2db7cef12ba09a51b0a7204fc81deeec. after waiting 0 ms 2024-12-03T23:14:40,746 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,iii,1733267676090.2db7cef12ba09a51b0a7204fc81deeec. 2024-12-03T23:14:40,750 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/default/test/2db7cef12ba09a51b0a7204fc81deeec/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-03T23:14:40,751 INFO [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,iii,1733267676090.2db7cef12ba09a51b0a7204fc81deeec. 2024-12-03T23:14:40,751 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 2db7cef12ba09a51b0a7204fc81deeec: Waiting for close lock at 1733267680746Running coprocessor pre-close hooks at 1733267680746Disabling compacts and flushes for region at 1733267680746Disabling writes for close at 1733267680746Writing region close event to WAL at 1733267680747 (+1 ms)Running coprocessor post-close hooks at 1733267680751 (+4 ms)Closed at 1733267680751 2024-12-03T23:14:40,751 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,iii,1733267676090.2db7cef12ba09a51b0a7204fc81deeec. 2024-12-03T23:14:40,751 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 2b7da3736c6b257b43fe66ea13807bc6, disabling compactions & flushes 2024-12-03T23:14:40,751 INFO [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,sss,1733267676090.2b7da3736c6b257b43fe66ea13807bc6. 2024-12-03T23:14:40,751 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,sss,1733267676090.2b7da3736c6b257b43fe66ea13807bc6. 2024-12-03T23:14:40,751 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,sss,1733267676090.2b7da3736c6b257b43fe66ea13807bc6. after waiting 0 ms 2024-12-03T23:14:40,751 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,sss,1733267676090.2b7da3736c6b257b43fe66ea13807bc6. 2024-12-03T23:14:40,755 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/default/test/2b7da3736c6b257b43fe66ea13807bc6/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-03T23:14:40,756 INFO [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,sss,1733267676090.2b7da3736c6b257b43fe66ea13807bc6. 2024-12-03T23:14:40,756 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 2b7da3736c6b257b43fe66ea13807bc6: Waiting for close lock at 1733267680751Running coprocessor pre-close hooks at 1733267680751Disabling compacts and flushes for region at 1733267680751Disabling writes for close at 1733267680751Writing region close event to WAL at 1733267680752 (+1 ms)Running coprocessor post-close hooks at 1733267680756 (+4 ms)Closed at 1733267680756 2024-12-03T23:14:40,756 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,sss,1733267676090.2b7da3736c6b257b43fe66ea13807bc6. 2024-12-03T23:14:40,756 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 4e6706440af7674db97bb48a05858115, disabling compactions & flushes 2024-12-03T23:14:40,756 INFO [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,mmm,1733267676090.4e6706440af7674db97bb48a05858115. 2024-12-03T23:14:40,756 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,mmm,1733267676090.4e6706440af7674db97bb48a05858115. 2024-12-03T23:14:40,756 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,mmm,1733267676090.4e6706440af7674db97bb48a05858115. after waiting 0 ms 2024-12-03T23:14:40,756 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,mmm,1733267676090.4e6706440af7674db97bb48a05858115. 2024-12-03T23:14:40,760 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/default/test/4e6706440af7674db97bb48a05858115/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-03T23:14:40,761 INFO [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,mmm,1733267676090.4e6706440af7674db97bb48a05858115. 2024-12-03T23:14:40,761 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 4e6706440af7674db97bb48a05858115: Waiting for close lock at 1733267680756Running coprocessor pre-close hooks at 1733267680756Disabling compacts and flushes for region at 1733267680756Disabling writes for close at 1733267680756Writing region close event to WAL at 1733267680756Running coprocessor post-close hooks at 1733267680761 (+5 ms)Closed at 1733267680761 2024-12-03T23:14:40,761 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,mmm,1733267676090.4e6706440af7674db97bb48a05858115. 2024-12-03T23:14:40,761 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 212a9963c9cddede26a4eb521cff5254, disabling compactions & flushes 2024-12-03T23:14:40,761 INFO [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,eee,1733267676090.212a9963c9cddede26a4eb521cff5254. 2024-12-03T23:14:40,761 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,eee,1733267676090.212a9963c9cddede26a4eb521cff5254. 2024-12-03T23:14:40,761 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,eee,1733267676090.212a9963c9cddede26a4eb521cff5254. after waiting 0 ms 2024-12-03T23:14:40,761 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,eee,1733267676090.212a9963c9cddede26a4eb521cff5254. 2024-12-03T23:14:40,765 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/default/test/212a9963c9cddede26a4eb521cff5254/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-03T23:14:40,765 INFO [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,eee,1733267676090.212a9963c9cddede26a4eb521cff5254. 2024-12-03T23:14:40,765 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 212a9963c9cddede26a4eb521cff5254: Waiting for close lock at 1733267680761Running coprocessor pre-close hooks at 1733267680761Disabling compacts and flushes for region at 1733267680761Disabling writes for close at 1733267680761Writing region close event to WAL at 1733267680761Running coprocessor post-close hooks at 1733267680765 (+4 ms)Closed at 1733267680765 2024-12-03T23:14:40,765 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,eee,1733267676090.212a9963c9cddede26a4eb521cff5254. 2024-12-03T23:14:40,765 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing f71f90ef2d2f59e885f2ad6ca1a99596, disabling compactions & flushes 2024-12-03T23:14:40,765 INFO [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,bbb,1733267676090.f71f90ef2d2f59e885f2ad6ca1a99596. 2024-12-03T23:14:40,765 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,bbb,1733267676090.f71f90ef2d2f59e885f2ad6ca1a99596. 2024-12-03T23:14:40,766 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,bbb,1733267676090.f71f90ef2d2f59e885f2ad6ca1a99596. after waiting 0 ms 2024-12-03T23:14:40,766 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,bbb,1733267676090.f71f90ef2d2f59e885f2ad6ca1a99596. 2024-12-03T23:14:40,769 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/data/default/test/f71f90ef2d2f59e885f2ad6ca1a99596/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-03T23:14:40,770 INFO [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,bbb,1733267676090.f71f90ef2d2f59e885f2ad6ca1a99596. 2024-12-03T23:14:40,770 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for f71f90ef2d2f59e885f2ad6ca1a99596: Waiting for close lock at 1733267680765Running coprocessor pre-close hooks at 1733267680765Disabling compacts and flushes for region at 1733267680765Disabling writes for close at 1733267680766 (+1 ms)Writing region close event to WAL at 1733267680766Running coprocessor post-close hooks at 1733267680770 (+4 ms)Closed at 1733267680770 2024-12-03T23:14:40,770 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,bbb,1733267676090.f71f90ef2d2f59e885f2ad6ca1a99596. 2024-12-03T23:14:40,857 INFO [regionserver/5f9d64419733:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-03T23:14:40,857 INFO [regionserver/5f9d64419733:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-03T23:14:40,869 INFO [RS:1;5f9d64419733:44899 {}] regionserver.HRegionServer(976): stopping server 5f9d64419733,44899,1733267673202; all regions closed. 2024-12-03T23:14:40,869 INFO [RS:2;5f9d64419733:41167 {}] regionserver.HRegionServer(976): stopping server 5f9d64419733,41167,1733267673270; all regions closed. 2024-12-03T23:14:40,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44333 is added to blk_1073741865_1041 (size=4192) 2024-12-03T23:14:40,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45029 is added to blk_1073741865_1041 (size=4192) 2024-12-03T23:14:40,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38883 is added to blk_1073741834_1010 (size=5194) 2024-12-03T23:14:40,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44333 is added to blk_1073741834_1010 (size=5194) 2024-12-03T23:14:40,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38883 is added to blk_1073741865_1041 (size=4192) 2024-12-03T23:14:40,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45029 is added to blk_1073741834_1010 (size=5194) 2024-12-03T23:14:40,878 DEBUG [RS:2;5f9d64419733:41167 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/oldWALs 2024-12-03T23:14:40,878 INFO [RS:2;5f9d64419733:41167 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 5f9d64419733%2C41167%2C1733267673270.meta:.meta(num 1733267679180) 2024-12-03T23:14:40,878 DEBUG [RS:1;5f9d64419733:44899 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/oldWALs 2024-12-03T23:14:40,878 INFO [RS:1;5f9d64419733:44899 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 5f9d64419733%2C44899%2C1733267673202:(num 1733267675041) 2024-12-03T23:14:40,878 DEBUG [RS:1;5f9d64419733:44899 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T23:14:40,878 INFO [RS:1;5f9d64419733:44899 {}] regionserver.LeaseManager(133): Closed leases 2024-12-03T23:14:40,878 INFO [RS:1;5f9d64419733:44899 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-03T23:14:40,879 INFO [RS:1;5f9d64419733:44899 {}] hbase.ChoreService(370): Chore service for: regionserver/5f9d64419733:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-03T23:14:40,879 INFO [RS:1;5f9d64419733:44899 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-03T23:14:40,879 INFO [RS:1;5f9d64419733:44899 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-03T23:14:40,879 INFO [RS:1;5f9d64419733:44899 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-03T23:14:40,879 INFO [RS:1;5f9d64419733:44899 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-03T23:14:40,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44333 is added to blk_1073741835_1011 (size=7074) 2024-12-03T23:14:40,881 INFO [regionserver/5f9d64419733:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-03T23:14:40,882 WARN [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] util.Threads(127): sleep interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.util.Threads.sleep(Threads.java:125) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportRegionStateTransition(HRegionServer.java:2374) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2257) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T23:14:40,882 WARN [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] handler.AssignRegionHandler(168): Fatal error occurred while opening region test,jjj,1733267676090.384b2e2791a8bf699cb7996e1aa7469c., aborting... java.io.IOException: Failed to report opened region to master: test,jjj,1733267676090.384b2e2791a8bf699cb7996e1aa7469c. at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2261) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T23:14:40,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38883 is added to blk_1073741835_1011 (size=7074) 2024-12-03T23:14:40,882 ERROR [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionServer(2440): ***** ABORTING region server 5f9d64419733,44899,1733267673202: Failed to open region test,jjj,1733267676090.384b2e2791a8bf699cb7996e1aa7469c. and can not recover ***** java.io.IOException: Failed to report opened region to master: test,jjj,1733267676090.384b2e2791a8bf699cb7996e1aa7469c. at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2261) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T23:14:40,882 ERROR [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-12-03T23:14:40,883 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-12-03T23:14:40,883 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-12-03T23:14:40,883 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-12-03T23:14:40,883 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-12-03T23:14:40,883 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 483436576 }, "NonHeapMemoryUsage": { "committed": 162463744, "init": 7667712, "max": -1, "used": 159786120 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [], "beans": [], "beans": [] } 2024-12-03T23:14:40,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45029 is added to blk_1073741835_1011 (size=7074) 2024-12-03T23:14:40,885 INFO [RS:1;5f9d64419733:44899 {}] executor.ExecutorService(108): Executor-11-RS_OPEN_REGION-regionserver/5f9d64419733:0 had [Event #36 of type M_RS_OPEN_REGION (class org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler), Event #37 of type M_RS_OPEN_REGION (class org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler), Event #38 of type M_RS_OPEN_REGION (class org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler)] on shutdown 2024-12-03T23:14:40,885 INFO [RS:1;5f9d64419733:44899 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:44899 2024-12-03T23:14:40,887 DEBUG [RS:2;5f9d64419733:41167 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/oldWALs 2024-12-03T23:14:40,887 INFO [RS:2;5f9d64419733:41167 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 5f9d64419733%2C41167%2C1733267673270:(num 1733267675050) 2024-12-03T23:14:40,887 DEBUG [RS:2;5f9d64419733:41167 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T23:14:40,887 INFO [RS:2;5f9d64419733:41167 {}] regionserver.LeaseManager(133): Closed leases 2024-12-03T23:14:40,887 INFO [RS:2;5f9d64419733:41167 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-03T23:14:40,887 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44899-0x1017a9bb4a60002, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/5f9d64419733,44899,1733267673202 2024-12-03T23:14:40,887 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44117-0x1017a9bb4a60000, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-03T23:14:40,887 INFO [RS:2;5f9d64419733:41167 {}] hbase.ChoreService(370): Chore service for: regionserver/5f9d64419733:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-03T23:14:40,888 INFO [RS:2;5f9d64419733:41167 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-03T23:14:40,888 INFO [regionserver/5f9d64419733:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-03T23:14:40,888 INFO [RS:1;5f9d64419733:44899 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-03T23:14:40,888 INFO [RS:2;5f9d64419733:41167 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41167 2024-12-03T23:14:40,890 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [5f9d64419733,44899,1733267673202] 2024-12-03T23:14:40,891 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41167-0x1017a9bb4a60003, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/5f9d64419733,41167,1733267673270 2024-12-03T23:14:40,891 INFO [RS:2;5f9d64419733:41167 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-03T23:14:40,891 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44117-0x1017a9bb4a60000, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-03T23:14:40,893 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/5f9d64419733,44899,1733267673202 already deleted, retry=false 2024-12-03T23:14:40,893 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 5f9d64419733,44899,1733267673202 expired; onlineServers=1 2024-12-03T23:14:40,894 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [5f9d64419733,41167,1733267673270] 2024-12-03T23:14:40,895 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/5f9d64419733,41167,1733267673270 already deleted, retry=false 2024-12-03T23:14:40,895 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 5f9d64419733,41167,1733267673270 expired; onlineServers=0 2024-12-03T23:14:40,896 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '5f9d64419733,44117,1733267672173' ***** 2024-12-03T23:14:40,896 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-03T23:14:40,896 INFO [M:0;5f9d64419733:44117 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-03T23:14:40,896 INFO [M:0;5f9d64419733:44117 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-03T23:14:40,896 DEBUG [M:0;5f9d64419733:44117 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-03T23:14:40,896 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-03T23:14:40,896 DEBUG [M:0;5f9d64419733:44117 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-03T23:14:40,896 DEBUG [master/5f9d64419733:0:becomeActiveMaster-HFileCleaner.large.0-1733267674580 {}] cleaner.HFileCleaner(306): Exit Thread[master/5f9d64419733:0:becomeActiveMaster-HFileCleaner.large.0-1733267674580,5,FailOnTimeoutGroup] 2024-12-03T23:14:40,896 DEBUG [master/5f9d64419733:0:becomeActiveMaster-HFileCleaner.small.0-1733267674589 {}] cleaner.HFileCleaner(306): Exit Thread[master/5f9d64419733:0:becomeActiveMaster-HFileCleaner.small.0-1733267674589,5,FailOnTimeoutGroup] 2024-12-03T23:14:40,896 INFO [M:0;5f9d64419733:44117 {}] hbase.ChoreService(370): Chore service for: master/5f9d64419733:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-03T23:14:40,896 INFO [M:0;5f9d64419733:44117 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-03T23:14:40,897 DEBUG [M:0;5f9d64419733:44117 {}] master.HMaster(1795): Stopping service threads 2024-12-03T23:14:40,897 INFO [M:0;5f9d64419733:44117 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-03T23:14:40,897 INFO [M:0;5f9d64419733:44117 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T23:14:40,897 INFO [M:0;5f9d64419733:44117 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-03T23:14:40,897 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-03T23:14:40,899 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44117-0x1017a9bb4a60000, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-03T23:14:40,899 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44117-0x1017a9bb4a60000, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T23:14:40,899 DEBUG [M:0;5f9d64419733:44117 {}] zookeeper.ZKUtil(347): master:44117-0x1017a9bb4a60000, quorum=127.0.0.1:62825, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-03T23:14:40,899 WARN [M:0;5f9d64419733:44117 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-03T23:14:40,899 INFO [M:0;5f9d64419733:44117 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/.lastflushedseqids 2024-12-03T23:14:40,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44333 is added to blk_1073741870_1046 (size=1560) 2024-12-03T23:14:40,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38883 is added to blk_1073741870_1046 (size=1560) 2024-12-03T23:14:40,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45029 is added to blk_1073741870_1046 (size=1560) 2024-12-03T23:14:40,915 INFO [M:0;5f9d64419733:44117 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-03T23:14:40,915 INFO [M:0;5f9d64419733:44117 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-03T23:14:40,916 DEBUG [M:0;5f9d64419733:44117 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-03T23:14:40,916 INFO [M:0;5f9d64419733:44117 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T23:14:40,916 DEBUG [M:0;5f9d64419733:44117 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T23:14:40,916 DEBUG [M:0;5f9d64419733:44117 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-03T23:14:40,916 DEBUG [M:0;5f9d64419733:44117 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T23:14:40,916 INFO [M:0;5f9d64419733:44117 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=248.05 KB heapSize=301.45 KB 2024-12-03T23:14:40,933 DEBUG [M:0;5f9d64419733:44117 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/0f943cf005ce4870bfe539eaf31bd516 is 82, key is hbase:meta,,1/info:regioninfo/1733267679497/Put/seqid=0 2024-12-03T23:14:40,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45029 is added to blk_1073741871_1047 (size=6071) 2024-12-03T23:14:40,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44333 is added to blk_1073741871_1047 (size=6071) 2024-12-03T23:14:40,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38883 is added to blk_1073741871_1047 (size=6071) 2024-12-03T23:14:40,942 INFO [M:0;5f9d64419733:44117 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=569 (bloomFilter=true), to=hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/0f943cf005ce4870bfe539eaf31bd516 2024-12-03T23:14:40,969 DEBUG [M:0;5f9d64419733:44117 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b242e0f6941a42cf9e3f2a904592fce5 is 2987, key is \x00\x00\x00\x00\x00\x00\x009/proc:d/1733267680634/Put/seqid=0 2024-12-03T23:14:40,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44333 is added to blk_1073741872_1048 (size=12797) 2024-12-03T23:14:40,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38883 is added to blk_1073741872_1048 (size=12797) 2024-12-03T23:14:40,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45029 is added to blk_1073741872_1048 (size=12797) 2024-12-03T23:14:40,978 INFO [M:0;5f9d64419733:44117 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=246.78 KB at sequenceid=569 (bloomFilter=true), to=hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b242e0f6941a42cf9e3f2a904592fce5 2024-12-03T23:14:40,991 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44899-0x1017a9bb4a60002, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T23:14:40,991 INFO [RS:1;5f9d64419733:44899 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-03T23:14:40,991 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44899-0x1017a9bb4a60002, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T23:14:40,991 INFO [RS:1;5f9d64419733:44899 {}] regionserver.HRegionServer(1031): Exiting; stopping=5f9d64419733,44899,1733267673202; zookeeper connection closed. 2024-12-03T23:14:40,992 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@430c08a3 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@430c08a3 2024-12-03T23:14:40,994 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41167-0x1017a9bb4a60003, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T23:14:40,994 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41167-0x1017a9bb4a60003, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T23:14:40,994 INFO [RS:2;5f9d64419733:41167 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-03T23:14:40,994 INFO [RS:2;5f9d64419733:41167 {}] regionserver.HRegionServer(1031): Exiting; stopping=5f9d64419733,41167,1733267673270; zookeeper connection closed. 2024-12-03T23:14:40,995 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@29934dd4 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@29934dd4 2024-12-03T23:14:40,995 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-12-03T23:14:41,000 DEBUG [M:0;5f9d64419733:44117 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/20eb8d474fcb401bb2efd86620908025 is 69, key is 5f9d64419733,41167,1733267673270/rs:state/1733267674743/Put/seqid=0 2024-12-03T23:14:41,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38883 is added to blk_1073741873_1049 (size=5442) 2024-12-03T23:14:41,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44333 is added to blk_1073741873_1049 (size=5442) 2024-12-03T23:14:41,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45029 is added to blk_1073741873_1049 (size=5442) 2024-12-03T23:14:41,009 INFO [M:0;5f9d64419733:44117 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=569 (bloomFilter=true), to=hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/20eb8d474fcb401bb2efd86620908025 2024-12-03T23:14:41,015 INFO [M:0;5f9d64419733:44117 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 20eb8d474fcb401bb2efd86620908025 2024-12-03T23:14:41,029 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T23:14:41,031 DEBUG [M:0;5f9d64419733:44117 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/5e4c4ee9ffe04054ae4aa70922e8276d is 52, key is load_balancer_on/state:d/1733267678466/Put/seqid=0 2024-12-03T23:14:41,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45029 is added to blk_1073741874_1050 (size=5058) 2024-12-03T23:14:41,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38883 is added to blk_1073741874_1050 (size=5058) 2024-12-03T23:14:41,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44333 is added to blk_1073741874_1050 (size=5058) 2024-12-03T23:14:41,041 INFO [M:0;5f9d64419733:44117 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=569 (bloomFilter=true), to=hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/5e4c4ee9ffe04054ae4aa70922e8276d 2024-12-03T23:14:41,049 DEBUG [M:0;5f9d64419733:44117 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/0f943cf005ce4870bfe539eaf31bd516 as hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/0f943cf005ce4870bfe539eaf31bd516 2024-12-03T23:14:41,057 INFO [M:0;5f9d64419733:44117 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/0f943cf005ce4870bfe539eaf31bd516, entries=14, sequenceid=569, filesize=5.9 K 2024-12-03T23:14:41,058 DEBUG [M:0;5f9d64419733:44117 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b242e0f6941a42cf9e3f2a904592fce5 as hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/b242e0f6941a42cf9e3f2a904592fce5 2024-12-03T23:14:41,064 INFO [M:0;5f9d64419733:44117 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/b242e0f6941a42cf9e3f2a904592fce5, entries=79, sequenceid=569, filesize=12.5 K 2024-12-03T23:14:41,066 DEBUG [M:0;5f9d64419733:44117 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/20eb8d474fcb401bb2efd86620908025 as hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/20eb8d474fcb401bb2efd86620908025 2024-12-03T23:14:41,072 INFO [M:0;5f9d64419733:44117 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 20eb8d474fcb401bb2efd86620908025 2024-12-03T23:14:41,073 INFO [M:0;5f9d64419733:44117 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/20eb8d474fcb401bb2efd86620908025, entries=3, sequenceid=569, filesize=5.3 K 2024-12-03T23:14:41,074 DEBUG [M:0;5f9d64419733:44117 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/5e4c4ee9ffe04054ae4aa70922e8276d as hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/5e4c4ee9ffe04054ae4aa70922e8276d 2024-12-03T23:14:41,081 INFO [M:0;5f9d64419733:44117 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44377/user/jenkins/test-data/2564fb6d-86ae-53b0-e92f-c26209ee0bc1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/5e4c4ee9ffe04054ae4aa70922e8276d, entries=1, sequenceid=569, filesize=4.9 K 2024-12-03T23:14:41,082 INFO [M:0;5f9d64419733:44117 {}] regionserver.HRegion(3140): Finished flush of dataSize ~248.05 KB/254004, heapSize ~301.38 KB/308616, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 166ms, sequenceid=569, compaction requested=false 2024-12-03T23:14:41,084 INFO [M:0;5f9d64419733:44117 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T23:14:41,084 DEBUG [M:0;5f9d64419733:44117 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733267680916Disabling compacts and flushes for region at 1733267680916Disabling writes for close at 1733267680916Obtaining lock to block concurrent updates at 1733267680916Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733267680916Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=254004, getHeapSize=308616, getOffHeapSize=0, getCellsCount=708 at 1733267680916Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733267680917 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733267680917Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733267680933 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733267680933Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733267680949 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733267680969 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733267680969Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733267680984 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733267681000 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733267681000Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1733267681016 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1733267681030 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1733267681030Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2681a6ef: reopening flushed file at 1733267681048 (+18 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@57286885: reopening flushed file at 1733267681057 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6f9c56fc: reopening flushed file at 1733267681065 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1797c6e2: reopening flushed file at 1733267681073 (+8 ms)Finished flush of dataSize ~248.05 KB/254004, heapSize ~301.38 KB/308616, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 166ms, sequenceid=569, compaction requested=false at 1733267681082 (+9 ms)Writing region close event to WAL at 1733267681084 (+2 ms)Closed at 1733267681084 2024-12-03T23:14:41,086 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'test' 2024-12-03T23:14:41,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38883 is added to blk_1073741830_1006 (size=291998) 2024-12-03T23:14:41,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44333 is added to blk_1073741830_1006 (size=291998) 2024-12-03T23:14:41,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45029 is added to blk_1073741830_1006 (size=291998) 2024-12-03T23:14:41,088 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-03T23:14:41,089 INFO [M:0;5f9d64419733:44117 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-03T23:14:41,089 INFO [M:0;5f9d64419733:44117 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:44117 2024-12-03T23:14:41,089 INFO [M:0;5f9d64419733:44117 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-03T23:14:41,192 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44117-0x1017a9bb4a60000, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T23:14:41,192 INFO [M:0;5f9d64419733:44117 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-03T23:14:41,192 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44117-0x1017a9bb4a60000, quorum=127.0.0.1:62825, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T23:14:41,197 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3c1f701{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-03T23:14:41,199 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3d599bb4{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-03T23:14:41,199 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-03T23:14:41,199 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@417efacb{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-03T23:14:41,199 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@290e2eec{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fb53328b-6122-8bbc-b1d5-834eeb8c019c/hadoop.log.dir/,STOPPED} 2024-12-03T23:14:41,202 WARN [BP-38275091-172.17.0.2-1733267668370 heartbeating to localhost/127.0.0.1:44377 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-03T23:14:41,202 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-03T23:14:41,202 WARN [BP-38275091-172.17.0.2-1733267668370 heartbeating to localhost/127.0.0.1:44377 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-38275091-172.17.0.2-1733267668370 (Datanode Uuid 6fb13a36-e8d9-4ed7-aafc-f06328038eb4) service to localhost/127.0.0.1:44377 2024-12-03T23:14:41,202 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-03T23:14:41,203 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fb53328b-6122-8bbc-b1d5-834eeb8c019c/cluster_8abe0f93-9472-57fc-9787-96cccde33366/data/data5/current/BP-38275091-172.17.0.2-1733267668370 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-03T23:14:41,204 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fb53328b-6122-8bbc-b1d5-834eeb8c019c/cluster_8abe0f93-9472-57fc-9787-96cccde33366/data/data6/current/BP-38275091-172.17.0.2-1733267668370 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-03T23:14:41,204 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-03T23:14:41,207 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@34fcb585{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-03T23:14:41,207 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@389bb6f7{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-03T23:14:41,207 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-03T23:14:41,207 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1b0a32ba{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-03T23:14:41,207 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@29652cf4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fb53328b-6122-8bbc-b1d5-834eeb8c019c/hadoop.log.dir/,STOPPED} 2024-12-03T23:14:41,209 WARN [BP-38275091-172.17.0.2-1733267668370 heartbeating to localhost/127.0.0.1:44377 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-03T23:14:41,209 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-03T23:14:41,209 WARN [BP-38275091-172.17.0.2-1733267668370 heartbeating to localhost/127.0.0.1:44377 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-38275091-172.17.0.2-1733267668370 (Datanode Uuid 3e73d5b6-a866-4f43-8324-0c742cc48899) service to localhost/127.0.0.1:44377 2024-12-03T23:14:41,209 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-03T23:14:41,210 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fb53328b-6122-8bbc-b1d5-834eeb8c019c/cluster_8abe0f93-9472-57fc-9787-96cccde33366/data/data3/current/BP-38275091-172.17.0.2-1733267668370 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-03T23:14:41,210 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fb53328b-6122-8bbc-b1d5-834eeb8c019c/cluster_8abe0f93-9472-57fc-9787-96cccde33366/data/data4/current/BP-38275091-172.17.0.2-1733267668370 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-03T23:14:41,210 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-03T23:14:41,217 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@79b30fb0{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-03T23:14:41,217 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@548cd725{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-03T23:14:41,218 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-03T23:14:41,218 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@474d518f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-03T23:14:41,218 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3a300759{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fb53328b-6122-8bbc-b1d5-834eeb8c019c/hadoop.log.dir/,STOPPED} 2024-12-03T23:14:41,220 WARN [BP-38275091-172.17.0.2-1733267668370 heartbeating to localhost/127.0.0.1:44377 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-03T23:14:41,220 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-03T23:14:41,220 WARN [BP-38275091-172.17.0.2-1733267668370 heartbeating to localhost/127.0.0.1:44377 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-38275091-172.17.0.2-1733267668370 (Datanode Uuid d3a8ad16-0743-44f9-9bd8-7ff859bebe67) service to localhost/127.0.0.1:44377 2024-12-03T23:14:41,220 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-03T23:14:41,220 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fb53328b-6122-8bbc-b1d5-834eeb8c019c/cluster_8abe0f93-9472-57fc-9787-96cccde33366/data/data1/current/BP-38275091-172.17.0.2-1733267668370 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-03T23:14:41,221 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fb53328b-6122-8bbc-b1d5-834eeb8c019c/cluster_8abe0f93-9472-57fc-9787-96cccde33366/data/data2/current/BP-38275091-172.17.0.2-1733267668370 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-03T23:14:41,221 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-03T23:14:41,234 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@64418798{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-03T23:14:41,234 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@11744524{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-03T23:14:41,234 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-03T23:14:41,235 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@595320c8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-03T23:14:41,235 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3912242{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fb53328b-6122-8bbc-b1d5-834eeb8c019c/hadoop.log.dir/,STOPPED} 2024-12-03T23:14:41,249 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-03T23:14:41,308 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down