2024-11-08 05:13:15,011 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 2024-11-08 05:13:15,022 main DEBUG Took 0.008776 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-08 05:13:15,022 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-08 05:13:15,023 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-08 05:13:15,023 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-08 05:13:15,024 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 05:13:15,031 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-08 05:13:15,042 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 05:13:15,043 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 05:13:15,044 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 05:13:15,044 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 05:13:15,045 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 05:13:15,045 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 05:13:15,046 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 05:13:15,046 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 05:13:15,046 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 05:13:15,047 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 05:13:15,047 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 05:13:15,048 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 05:13:15,048 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 05:13:15,048 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 05:13:15,049 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 05:13:15,049 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 05:13:15,050 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 05:13:15,050 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 05:13:15,050 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 05:13:15,051 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 05:13:15,051 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 05:13:15,051 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 05:13:15,052 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 05:13:15,052 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 05:13:15,053 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 05:13:15,053 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-08 05:13:15,055 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 05:13:15,056 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-08 05:13:15,058 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-08 05:13:15,058 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-08 05:13:15,059 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-08 05:13:15,059 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-08 05:13:15,067 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-08 05:13:15,069 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-08 05:13:15,071 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-08 05:13:15,071 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-08 05:13:15,072 main DEBUG createAppenders(={Console}) 2024-11-08 05:13:15,073 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 initialized 2024-11-08 05:13:15,073 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 2024-11-08 05:13:15,073 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 OK. 2024-11-08 05:13:15,074 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-08 05:13:15,074 main DEBUG OutputStream closed 2024-11-08 05:13:15,075 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-08 05:13:15,075 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-08 05:13:15,075 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@c88a337 OK 2024-11-08 05:13:15,141 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-08 05:13:15,143 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-08 05:13:15,144 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-08 05:13:15,145 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-08 05:13:15,146 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-08 05:13:15,146 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-08 05:13:15,146 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-08 05:13:15,146 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-08 05:13:15,147 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-08 05:13:15,147 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-08 05:13:15,147 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-08 05:13:15,148 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-08 05:13:15,148 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-08 05:13:15,148 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-08 05:13:15,149 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-08 05:13:15,149 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-08 05:13:15,149 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-08 05:13:15,150 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-08 05:13:15,152 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-08 05:13:15,152 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@13330ac6) with optional ClassLoader: null 2024-11-08 05:13:15,152 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-08 05:13:15,153 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@13330ac6] started OK. 2024-11-08T05:13:15,167 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.master.assignment.TestRollbackSCP timeout: 13 mins 2024-11-08 05:13:15,170 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-08 05:13:15,170 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-08T05:13:15,425 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/494744ca-67d2-5496-2c52-6b1334939ddf 2024-11-08T05:13:15,446 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=class org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$HMasterForTest, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-08T05:13:15,469 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/494744ca-67d2-5496-2c52-6b1334939ddf/cluster_b5b9bc45-3113-1e92-d4ad-d82b8cd072e6, deleteOnExit=true 2024-11-08T05:13:15,469 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-08T05:13:15,471 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/494744ca-67d2-5496-2c52-6b1334939ddf/test.cache.data in system properties and HBase conf 2024-11-08T05:13:15,472 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/494744ca-67d2-5496-2c52-6b1334939ddf/hadoop.tmp.dir in system properties and HBase conf 2024-11-08T05:13:15,473 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/494744ca-67d2-5496-2c52-6b1334939ddf/hadoop.log.dir in system properties and HBase conf 2024-11-08T05:13:15,473 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/494744ca-67d2-5496-2c52-6b1334939ddf/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-08T05:13:15,474 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/494744ca-67d2-5496-2c52-6b1334939ddf/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-08T05:13:15,474 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-08T05:13:15,556 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-08T05:13:15,648 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T05:13:15,652 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/494744ca-67d2-5496-2c52-6b1334939ddf/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-08T05:13:15,653 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/494744ca-67d2-5496-2c52-6b1334939ddf/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-08T05:13:15,654 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/494744ca-67d2-5496-2c52-6b1334939ddf/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-08T05:13:15,655 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/494744ca-67d2-5496-2c52-6b1334939ddf/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-08T05:13:15,655 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/494744ca-67d2-5496-2c52-6b1334939ddf/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-08T05:13:15,656 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/494744ca-67d2-5496-2c52-6b1334939ddf/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-08T05:13:15,657 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/494744ca-67d2-5496-2c52-6b1334939ddf/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-08T05:13:15,657 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/494744ca-67d2-5496-2c52-6b1334939ddf/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-08T05:13:15,658 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/494744ca-67d2-5496-2c52-6b1334939ddf/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-08T05:13:15,659 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/494744ca-67d2-5496-2c52-6b1334939ddf/nfs.dump.dir in system properties and HBase conf 2024-11-08T05:13:15,659 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/494744ca-67d2-5496-2c52-6b1334939ddf/java.io.tmpdir in system properties and HBase conf 2024-11-08T05:13:15,660 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/494744ca-67d2-5496-2c52-6b1334939ddf/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-08T05:13:15,660 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/494744ca-67d2-5496-2c52-6b1334939ddf/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-08T05:13:15,661 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/494744ca-67d2-5496-2c52-6b1334939ddf/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-08T05:13:16,790 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-08T05:13:16,871 INFO [Time-limited test {}] log.Log(170): Logging initialized @2589ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-08T05:13:16,961 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-08T05:13:17,052 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-08T05:13:17,077 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-08T05:13:17,077 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-08T05:13:17,079 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-08T05:13:17,091 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-08T05:13:17,095 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@265b24f1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/494744ca-67d2-5496-2c52-6b1334939ddf/hadoop.log.dir/,AVAILABLE} 2024-11-08T05:13:17,096 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@76beb053{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-08T05:13:17,276 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@376fa8f3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/494744ca-67d2-5496-2c52-6b1334939ddf/java.io.tmpdir/jetty-localhost-45445-hadoop-hdfs-3_4_1-tests_jar-_-any-12062104339391895121/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-08T05:13:17,284 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@672b88ea{HTTP/1.1, (http/1.1)}{localhost:45445} 2024-11-08T05:13:17,284 INFO [Time-limited test {}] server.Server(415): Started @3003ms 2024-11-08T05:13:17,869 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-08T05:13:17,876 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-08T05:13:17,877 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-08T05:13:17,878 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-08T05:13:17,878 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-08T05:13:17,879 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@739a104e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/494744ca-67d2-5496-2c52-6b1334939ddf/hadoop.log.dir/,AVAILABLE} 2024-11-08T05:13:17,879 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1c5d2af1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-08T05:13:17,978 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2226d6a6{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/494744ca-67d2-5496-2c52-6b1334939ddf/java.io.tmpdir/jetty-localhost-43749-hadoop-hdfs-3_4_1-tests_jar-_-any-5849375693832488042/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-08T05:13:17,979 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@1da1f773{HTTP/1.1, (http/1.1)}{localhost:43749} 2024-11-08T05:13:17,979 INFO [Time-limited test {}] server.Server(415): Started @3698ms 2024-11-08T05:13:18,027 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-08T05:13:18,135 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-08T05:13:18,142 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-08T05:13:18,144 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-08T05:13:18,144 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-08T05:13:18,144 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-08T05:13:18,145 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@34a3e1ff{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/494744ca-67d2-5496-2c52-6b1334939ddf/hadoop.log.dir/,AVAILABLE} 2024-11-08T05:13:18,146 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@12f16529{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-08T05:13:18,266 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7cfccf27{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/494744ca-67d2-5496-2c52-6b1334939ddf/java.io.tmpdir/jetty-localhost-45577-hadoop-hdfs-3_4_1-tests_jar-_-any-14111845877408942278/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-08T05:13:18,267 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6c41bbb7{HTTP/1.1, (http/1.1)}{localhost:45577} 2024-11-08T05:13:18,267 INFO [Time-limited test {}] server.Server(415): Started @3986ms 2024-11-08T05:13:18,270 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-08T05:13:18,308 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-08T05:13:18,315 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-08T05:13:18,329 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-08T05:13:18,329 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-08T05:13:18,330 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-08T05:13:18,332 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@11ebaa12{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/494744ca-67d2-5496-2c52-6b1334939ddf/hadoop.log.dir/,AVAILABLE} 2024-11-08T05:13:18,333 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@52cf89e6{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-08T05:13:18,434 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@67ddfd25{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/494744ca-67d2-5496-2c52-6b1334939ddf/java.io.tmpdir/jetty-localhost-42965-hadoop-hdfs-3_4_1-tests_jar-_-any-3093434732292260656/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-08T05:13:18,434 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@30e38263{HTTP/1.1, (http/1.1)}{localhost:42965} 2024-11-08T05:13:18,435 INFO [Time-limited test {}] server.Server(415): Started @4154ms 2024-11-08T05:13:18,437 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-08T05:13:20,201 WARN [Thread-125 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/494744ca-67d2-5496-2c52-6b1334939ddf/cluster_b5b9bc45-3113-1e92-d4ad-d82b8cd072e6/data/data4/current/BP-148118038-172.17.0.3-1731042796219/current, will proceed with Du for space computation calculation, 2024-11-08T05:13:20,201 WARN [Thread-124 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/494744ca-67d2-5496-2c52-6b1334939ddf/cluster_b5b9bc45-3113-1e92-d4ad-d82b8cd072e6/data/data3/current/BP-148118038-172.17.0.3-1731042796219/current, will proceed with Du for space computation calculation, 2024-11-08T05:13:20,201 WARN [Thread-123 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/494744ca-67d2-5496-2c52-6b1334939ddf/cluster_b5b9bc45-3113-1e92-d4ad-d82b8cd072e6/data/data2/current/BP-148118038-172.17.0.3-1731042796219/current, will proceed with Du for space computation calculation, 2024-11-08T05:13:20,201 WARN [Thread-122 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/494744ca-67d2-5496-2c52-6b1334939ddf/cluster_b5b9bc45-3113-1e92-d4ad-d82b8cd072e6/data/data1/current/BP-148118038-172.17.0.3-1731042796219/current, will proceed with Du for space computation calculation, 2024-11-08T05:13:20,281 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-08T05:13:20,281 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-08T05:13:20,319 WARN [Thread-142 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/494744ca-67d2-5496-2c52-6b1334939ddf/cluster_b5b9bc45-3113-1e92-d4ad-d82b8cd072e6/data/data5/current/BP-148118038-172.17.0.3-1731042796219/current, will proceed with Du for space computation calculation, 2024-11-08T05:13:20,333 WARN [Thread-143 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/494744ca-67d2-5496-2c52-6b1334939ddf/cluster_b5b9bc45-3113-1e92-d4ad-d82b8cd072e6/data/data6/current/BP-148118038-172.17.0.3-1731042796219/current, will proceed with Du for space computation calculation, 2024-11-08T05:13:20,367 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x6881a90a7f31a0d5 with lease ID 0x686306b8543a8d0d: Processing first storage report for DS-7382e2ea-12bb-4310-9c38-6f88a0ed4f48 from datanode DatanodeRegistration(127.0.0.1:37041, datanodeUuid=4cfcc222-3171-4d68-9431-a0e80dfd6ba6, infoPort=35281, infoSecurePort=0, ipcPort=37655, storageInfo=lv=-57;cid=testClusterID;nsid=1622511069;c=1731042796219) 2024-11-08T05:13:20,368 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x6881a90a7f31a0d5 with lease ID 0x686306b8543a8d0d: from storage DS-7382e2ea-12bb-4310-9c38-6f88a0ed4f48 node DatanodeRegistration(127.0.0.1:37041, datanodeUuid=4cfcc222-3171-4d68-9431-a0e80dfd6ba6, infoPort=35281, infoSecurePort=0, ipcPort=37655, storageInfo=lv=-57;cid=testClusterID;nsid=1622511069;c=1731042796219), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-08T05:13:20,369 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x8447632865c24007 with lease ID 0x686306b8543a8d0e: Processing first storage report for DS-097c58aa-51b2-4f6f-9fb5-10702640c273 from datanode DatanodeRegistration(127.0.0.1:41203, datanodeUuid=7daa6c49-f147-4f86-84ea-b92d941c350c, infoPort=43885, infoSecurePort=0, ipcPort=45397, storageInfo=lv=-57;cid=testClusterID;nsid=1622511069;c=1731042796219) 2024-11-08T05:13:20,369 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x8447632865c24007 with lease ID 0x686306b8543a8d0e: from storage DS-097c58aa-51b2-4f6f-9fb5-10702640c273 node DatanodeRegistration(127.0.0.1:41203, datanodeUuid=7daa6c49-f147-4f86-84ea-b92d941c350c, infoPort=43885, infoSecurePort=0, ipcPort=45397, storageInfo=lv=-57;cid=testClusterID;nsid=1622511069;c=1731042796219), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-08T05:13:20,369 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x6881a90a7f31a0d5 with lease ID 0x686306b8543a8d0d: Processing first storage report for DS-088152dd-c37e-463c-9232-a82e2573322d from datanode DatanodeRegistration(127.0.0.1:37041, datanodeUuid=4cfcc222-3171-4d68-9431-a0e80dfd6ba6, infoPort=35281, infoSecurePort=0, ipcPort=37655, storageInfo=lv=-57;cid=testClusterID;nsid=1622511069;c=1731042796219) 2024-11-08T05:13:20,370 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x6881a90a7f31a0d5 with lease ID 0x686306b8543a8d0d: from storage DS-088152dd-c37e-463c-9232-a82e2573322d node DatanodeRegistration(127.0.0.1:37041, datanodeUuid=4cfcc222-3171-4d68-9431-a0e80dfd6ba6, infoPort=35281, infoSecurePort=0, ipcPort=37655, storageInfo=lv=-57;cid=testClusterID;nsid=1622511069;c=1731042796219), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-08T05:13:20,370 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x8447632865c24007 with lease ID 0x686306b8543a8d0e: Processing first storage report for DS-8397f2d5-660e-46f0-bf9e-b36ea05d4c84 from datanode DatanodeRegistration(127.0.0.1:41203, datanodeUuid=7daa6c49-f147-4f86-84ea-b92d941c350c, infoPort=43885, infoSecurePort=0, ipcPort=45397, storageInfo=lv=-57;cid=testClusterID;nsid=1622511069;c=1731042796219) 2024-11-08T05:13:20,370 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x8447632865c24007 with lease ID 0x686306b8543a8d0e: from storage DS-8397f2d5-660e-46f0-bf9e-b36ea05d4c84 node DatanodeRegistration(127.0.0.1:41203, datanodeUuid=7daa6c49-f147-4f86-84ea-b92d941c350c, infoPort=43885, infoSecurePort=0, ipcPort=45397, storageInfo=lv=-57;cid=testClusterID;nsid=1622511069;c=1731042796219), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-08T05:13:20,445 WARN [Thread-103 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-08T05:13:20,460 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xc8975997024bfcfa with lease ID 0x686306b8543a8d0f: Processing first storage report for DS-580c23be-6f59-4a94-8328-84ad55cf144b from datanode DatanodeRegistration(127.0.0.1:34565, datanodeUuid=a7d9b40c-b323-4028-a313-57a6ed1b1f07, infoPort=34771, infoSecurePort=0, ipcPort=36365, storageInfo=lv=-57;cid=testClusterID;nsid=1622511069;c=1731042796219) 2024-11-08T05:13:20,460 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc8975997024bfcfa with lease ID 0x686306b8543a8d0f: from storage DS-580c23be-6f59-4a94-8328-84ad55cf144b node DatanodeRegistration(127.0.0.1:34565, datanodeUuid=a7d9b40c-b323-4028-a313-57a6ed1b1f07, infoPort=34771, infoSecurePort=0, ipcPort=36365, storageInfo=lv=-57;cid=testClusterID;nsid=1622511069;c=1731042796219), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-08T05:13:20,461 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xc8975997024bfcfa with lease ID 0x686306b8543a8d0f: Processing first storage report for DS-cdbee200-6888-478f-b9e4-25d29a03e2be from datanode DatanodeRegistration(127.0.0.1:34565, datanodeUuid=a7d9b40c-b323-4028-a313-57a6ed1b1f07, infoPort=34771, infoSecurePort=0, ipcPort=36365, storageInfo=lv=-57;cid=testClusterID;nsid=1622511069;c=1731042796219) 2024-11-08T05:13:20,461 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc8975997024bfcfa with lease ID 0x686306b8543a8d0f: from storage DS-cdbee200-6888-478f-b9e4-25d29a03e2be node DatanodeRegistration(127.0.0.1:34565, datanodeUuid=a7d9b40c-b323-4028-a313-57a6ed1b1f07, infoPort=34771, infoSecurePort=0, ipcPort=36365, storageInfo=lv=-57;cid=testClusterID;nsid=1622511069;c=1731042796219), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-08T05:13:20,492 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/494744ca-67d2-5496-2c52-6b1334939ddf 2024-11-08T05:13:20,684 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/494744ca-67d2-5496-2c52-6b1334939ddf/cluster_b5b9bc45-3113-1e92-d4ad-d82b8cd072e6/zookeeper_0, clientPort=57355, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/494744ca-67d2-5496-2c52-6b1334939ddf/cluster_b5b9bc45-3113-1e92-d4ad-d82b8cd072e6/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/494744ca-67d2-5496-2c52-6b1334939ddf/cluster_b5b9bc45-3113-1e92-d4ad-d82b8cd072e6/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-08T05:13:20,722 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=57355 2024-11-08T05:13:20,741 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T05:13:20,745 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T05:13:21,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741825_1001 (size=7) 2024-11-08T05:13:21,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741825_1001 (size=7) 2024-11-08T05:13:21,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34565 is added to blk_1073741825_1001 (size=7) 2024-11-08T05:13:21,490 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c with version=8 2024-11-08T05:13:21,490 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/hbase-staging 2024-11-08T05:13:21,642 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-08T05:13:22,066 INFO [Time-limited test {}] client.ConnectionUtils(128): master/d1ba675678c9:0 server-side Connection retries=45 2024-11-08T05:13:22,079 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T05:13:22,080 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-08T05:13:22,115 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-08T05:13:22,115 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T05:13:22,115 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-08T05:13:22,313 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-08T05:13:22,394 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-08T05:13:22,410 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-08T05:13:22,415 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-08T05:13:22,452 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 3174 (auto-detected) 2024-11-08T05:13:22,454 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-11-08T05:13:22,491 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:37587 2024-11-08T05:13:22,529 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:37587 connecting to ZooKeeper ensemble=127.0.0.1:57355 2024-11-08T05:13:22,638 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:375870x0, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-08T05:13:22,642 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:37587-0x10118dcefab0000 connected 2024-11-08T05:13:22,736 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T05:13:22,739 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T05:13:22,751 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37587-0x10118dcefab0000, quorum=127.0.0.1:57355, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T05:13:22,755 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c, hbase.cluster.distributed=false 2024-11-08T05:13:22,775 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37587-0x10118dcefab0000, quorum=127.0.0.1:57355, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-08T05:13:22,781 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37587 2024-11-08T05:13:22,781 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37587 2024-11-08T05:13:22,782 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37587 2024-11-08T05:13:22,783 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37587 2024-11-08T05:13:22,783 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37587 2024-11-08T05:13:22,892 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/d1ba675678c9:0 server-side Connection retries=45 2024-11-08T05:13:22,895 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T05:13:22,895 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-08T05:13:22,896 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-08T05:13:22,896 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T05:13:22,896 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-08T05:13:22,900 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-08T05:13:22,904 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-08T05:13:22,905 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:46753 2024-11-08T05:13:22,907 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:46753 connecting to ZooKeeper ensemble=127.0.0.1:57355 2024-11-08T05:13:22,909 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T05:13:22,912 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T05:13:22,924 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:467530x0, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-08T05:13:22,925 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:467530x0, quorum=127.0.0.1:57355, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T05:13:22,925 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:46753-0x10118dcefab0001 connected 2024-11-08T05:13:22,929 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-08T05:13:22,937 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-08T05:13:22,940 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46753-0x10118dcefab0001, quorum=127.0.0.1:57355, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-08T05:13:22,946 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46753-0x10118dcefab0001, quorum=127.0.0.1:57355, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-08T05:13:22,947 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46753 2024-11-08T05:13:22,948 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46753 2024-11-08T05:13:22,950 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46753 2024-11-08T05:13:22,952 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46753 2024-11-08T05:13:22,953 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46753 2024-11-08T05:13:22,970 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/d1ba675678c9:0 server-side Connection retries=45 2024-11-08T05:13:22,971 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T05:13:22,971 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-08T05:13:22,971 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-08T05:13:22,971 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T05:13:22,972 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-08T05:13:22,972 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-08T05:13:22,972 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-08T05:13:22,973 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:41485 2024-11-08T05:13:22,975 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:41485 connecting to ZooKeeper ensemble=127.0.0.1:57355 2024-11-08T05:13:22,976 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T05:13:22,979 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T05:13:22,994 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:414850x0, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-08T05:13:22,995 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:414850x0, quorum=127.0.0.1:57355, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T05:13:22,995 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:41485-0x10118dcefab0002 connected 2024-11-08T05:13:22,996 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-08T05:13:22,998 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-08T05:13:22,999 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41485-0x10118dcefab0002, quorum=127.0.0.1:57355, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-08T05:13:23,002 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41485-0x10118dcefab0002, quorum=127.0.0.1:57355, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-08T05:13:23,002 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41485 2024-11-08T05:13:23,003 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41485 2024-11-08T05:13:23,003 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41485 2024-11-08T05:13:23,004 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41485 2024-11-08T05:13:23,004 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41485 2024-11-08T05:13:23,020 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/d1ba675678c9:0 server-side Connection retries=45 2024-11-08T05:13:23,020 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T05:13:23,020 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-08T05:13:23,021 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-08T05:13:23,021 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T05:13:23,021 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-08T05:13:23,021 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-08T05:13:23,021 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-08T05:13:23,022 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:39291 2024-11-08T05:13:23,024 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:39291 connecting to ZooKeeper ensemble=127.0.0.1:57355 2024-11-08T05:13:23,025 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T05:13:23,027 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T05:13:23,040 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:392910x0, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-08T05:13:23,041 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:39291-0x10118dcefab0003 connected 2024-11-08T05:13:23,041 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:392910x0, quorum=127.0.0.1:57355, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T05:13:23,042 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-08T05:13:23,043 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-08T05:13:23,044 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39291-0x10118dcefab0003, quorum=127.0.0.1:57355, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-08T05:13:23,046 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39291-0x10118dcefab0003, quorum=127.0.0.1:57355, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-08T05:13:23,047 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39291 2024-11-08T05:13:23,047 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39291 2024-11-08T05:13:23,048 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39291 2024-11-08T05:13:23,048 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39291 2024-11-08T05:13:23,053 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39291 2024-11-08T05:13:23,066 DEBUG [M:0;d1ba675678c9:37587 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;d1ba675678c9:37587 2024-11-08T05:13:23,067 INFO [master/d1ba675678c9:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/d1ba675678c9,37587,1731042801796 2024-11-08T05:13:23,082 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39291-0x10118dcefab0003, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T05:13:23,082 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37587-0x10118dcefab0000, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T05:13:23,082 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41485-0x10118dcefab0002, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T05:13:23,082 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46753-0x10118dcefab0001, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T05:13:23,085 DEBUG [master/d1ba675678c9:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37587-0x10118dcefab0000, quorum=127.0.0.1:57355, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/d1ba675678c9,37587,1731042801796 2024-11-08T05:13:23,114 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41485-0x10118dcefab0002, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-08T05:13:23,114 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46753-0x10118dcefab0001, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-08T05:13:23,114 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37587-0x10118dcefab0000, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T05:13:23,114 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39291-0x10118dcefab0003, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-08T05:13:23,114 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41485-0x10118dcefab0002, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T05:13:23,114 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39291-0x10118dcefab0003, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T05:13:23,114 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46753-0x10118dcefab0001, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T05:13:23,115 DEBUG [master/d1ba675678c9:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37587-0x10118dcefab0000, quorum=127.0.0.1:57355, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-08T05:13:23,116 INFO [master/d1ba675678c9:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/d1ba675678c9,37587,1731042801796 from backup master directory 2024-11-08T05:13:23,124 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39291-0x10118dcefab0003, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T05:13:23,124 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37587-0x10118dcefab0000, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/d1ba675678c9,37587,1731042801796 2024-11-08T05:13:23,124 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41485-0x10118dcefab0002, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T05:13:23,124 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46753-0x10118dcefab0001, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T05:13:23,125 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37587-0x10118dcefab0000, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T05:13:23,125 WARN [master/d1ba675678c9:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-08T05:13:23,126 INFO [master/d1ba675678c9:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=d1ba675678c9,37587,1731042801796 2024-11-08T05:13:23,127 INFO [master/d1ba675678c9:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-08T05:13:23,129 INFO [master/d1ba675678c9:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-08T05:13:23,186 DEBUG [master/d1ba675678c9:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/hbase.id] with ID: 22414151-4211-4121-a8ec-5820592eaa4e 2024-11-08T05:13:23,186 DEBUG [master/d1ba675678c9:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/.tmp/hbase.id 2024-11-08T05:13:23,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34565 is added to blk_1073741826_1002 (size=42) 2024-11-08T05:13:23,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741826_1002 (size=42) 2024-11-08T05:13:23,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741826_1002 (size=42) 2024-11-08T05:13:23,203 DEBUG [master/d1ba675678c9:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/.tmp/hbase.id]:[hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/hbase.id] 2024-11-08T05:13:23,252 INFO [master/d1ba675678c9:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T05:13:23,258 INFO [master/d1ba675678c9:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-08T05:13:23,284 INFO [master/d1ba675678c9:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 22ms. 2024-11-08T05:13:23,293 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39291-0x10118dcefab0003, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T05:13:23,293 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37587-0x10118dcefab0000, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T05:13:23,293 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41485-0x10118dcefab0002, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T05:13:23,293 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46753-0x10118dcefab0001, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T05:13:23,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741827_1003 (size=196) 2024-11-08T05:13:23,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34565 is added to blk_1073741827_1003 (size=196) 2024-11-08T05:13:23,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741827_1003 (size=196) 2024-11-08T05:13:23,329 INFO [master/d1ba675678c9:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T05:13:23,330 INFO [master/d1ba675678c9:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-08T05:13:23,345 DEBUG [master/d1ba675678c9:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T05:13:23,348 INFO [master/d1ba675678c9:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-08T05:13:23,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741828_1004 (size=1189) 2024-11-08T05:13:23,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741828_1004 (size=1189) 2024-11-08T05:13:23,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34565 is added to blk_1073741828_1004 (size=1189) 2024-11-08T05:13:23,397 INFO [master/d1ba675678c9:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/MasterData/data/master/store 2024-11-08T05:13:23,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34565 is added to blk_1073741829_1005 (size=34) 2024-11-08T05:13:23,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741829_1005 (size=34) 2024-11-08T05:13:23,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741829_1005 (size=34) 2024-11-08T05:13:23,423 INFO [master/d1ba675678c9:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-08T05:13:23,427 DEBUG [master/d1ba675678c9:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T05:13:23,428 DEBUG [master/d1ba675678c9:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-08T05:13:23,428 INFO [master/d1ba675678c9:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T05:13:23,428 DEBUG [master/d1ba675678c9:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T05:13:23,430 DEBUG [master/d1ba675678c9:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-08T05:13:23,430 DEBUG [master/d1ba675678c9:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T05:13:23,430 INFO [master/d1ba675678c9:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T05:13:23,431 DEBUG [master/d1ba675678c9:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731042803428Disabling compacts and flushes for region at 1731042803428Disabling writes for close at 1731042803430 (+2 ms)Writing region close event to WAL at 1731042803430Closed at 1731042803430 2024-11-08T05:13:23,434 WARN [master/d1ba675678c9:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/MasterData/data/master/store/.initializing 2024-11-08T05:13:23,434 DEBUG [master/d1ba675678c9:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/MasterData/WALs/d1ba675678c9,37587,1731042801796 2024-11-08T05:13:23,442 INFO [master/d1ba675678c9:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-08T05:13:23,456 INFO [master/d1ba675678c9:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d1ba675678c9%2C37587%2C1731042801796, suffix=, logDir=hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/MasterData/WALs/d1ba675678c9,37587,1731042801796, archiveDir=hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/MasterData/oldWALs, maxLogs=10 2024-11-08T05:13:23,478 DEBUG [master/d1ba675678c9:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/MasterData/WALs/d1ba675678c9,37587,1731042801796/d1ba675678c9%2C37587%2C1731042801796.1731042803460, exclude list is [], retry=0 2024-11-08T05:13:23,481 WARN [IPC Server handler 4 on default port 45429 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-08T05:13:23,482 WARN [IPC Server handler 4 on default port 45429 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-08T05:13:23,482 WARN [IPC Server handler 4 on default port 45429 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-08T05:13:23,495 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37041,DS-7382e2ea-12bb-4310-9c38-6f88a0ed4f48,DISK] 2024-11-08T05:13:23,495 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41203,DS-097c58aa-51b2-4f6f-9fb5-10702640c273,DISK] 2024-11-08T05:13:23,498 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-08T05:13:23,533 INFO [master/d1ba675678c9:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/MasterData/WALs/d1ba675678c9,37587,1731042801796/d1ba675678c9%2C37587%2C1731042801796.1731042803460 2024-11-08T05:13:23,534 DEBUG [master/d1ba675678c9:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43885:43885),(127.0.0.1/127.0.0.1:35281:35281)] 2024-11-08T05:13:23,535 DEBUG [master/d1ba675678c9:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-08T05:13:23,535 DEBUG [master/d1ba675678c9:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T05:13:23,538 DEBUG [master/d1ba675678c9:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T05:13:23,539 DEBUG [master/d1ba675678c9:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T05:13:23,578 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-08T05:13:23,601 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-08T05:13:23,604 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T05:13:23,607 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T05:13:23,607 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-08T05:13:23,611 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-08T05:13:23,611 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T05:13:23,612 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T05:13:23,613 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-08T05:13:23,616 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-08T05:13:23,616 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T05:13:23,617 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T05:13:23,617 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-08T05:13:23,621 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-08T05:13:23,621 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T05:13:23,623 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T05:13:23,623 DEBUG [master/d1ba675678c9:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T05:13:23,628 DEBUG [master/d1ba675678c9:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-08T05:13:23,630 DEBUG [master/d1ba675678c9:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-08T05:13:23,636 DEBUG [master/d1ba675678c9:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T05:13:23,637 DEBUG [master/d1ba675678c9:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T05:13:23,640 DEBUG [master/d1ba675678c9:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-08T05:13:23,644 DEBUG [master/d1ba675678c9:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T05:13:23,648 DEBUG [master/d1ba675678c9:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T05:13:23,649 INFO [master/d1ba675678c9:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67441644, jitterRate=0.004958808422088623}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-08T05:13:23,656 DEBUG [master/d1ba675678c9:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731042803554Initializing all the Stores at 1731042803556 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731042803557 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731042803557Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731042803558 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731042803558Cleaning up temporary data from old regions at 1731042803637 (+79 ms)Region opened successfully at 1731042803656 (+19 ms) 2024-11-08T05:13:23,658 INFO [master/d1ba675678c9:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-08T05:13:23,693 DEBUG [master/d1ba675678c9:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3dfd2550, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d1ba675678c9/172.17.0.3:0 2024-11-08T05:13:23,731 INFO [master/d1ba675678c9:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-08T05:13:23,742 INFO [master/d1ba675678c9:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-08T05:13:23,742 INFO [master/d1ba675678c9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-08T05:13:23,745 INFO [master/d1ba675678c9:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-08T05:13:23,746 INFO [master/d1ba675678c9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-08T05:13:23,751 INFO [master/d1ba675678c9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-08T05:13:23,751 INFO [master/d1ba675678c9:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-08T05:13:23,781 INFO [master/d1ba675678c9:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-08T05:13:23,792 DEBUG [master/d1ba675678c9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37587-0x10118dcefab0000, quorum=127.0.0.1:57355, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-08T05:13:23,840 DEBUG [master/d1ba675678c9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-08T05:13:23,842 INFO [master/d1ba675678c9:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-08T05:13:23,844 DEBUG [master/d1ba675678c9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37587-0x10118dcefab0000, quorum=127.0.0.1:57355, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-08T05:13:23,850 DEBUG [master/d1ba675678c9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-08T05:13:23,853 INFO [master/d1ba675678c9:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-08T05:13:23,856 DEBUG [master/d1ba675678c9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37587-0x10118dcefab0000, quorum=127.0.0.1:57355, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-08T05:13:23,867 DEBUG [master/d1ba675678c9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-08T05:13:23,869 DEBUG [master/d1ba675678c9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37587-0x10118dcefab0000, quorum=127.0.0.1:57355, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-08T05:13:23,882 DEBUG [master/d1ba675678c9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-08T05:13:23,901 DEBUG [master/d1ba675678c9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37587-0x10118dcefab0000, quorum=127.0.0.1:57355, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-08T05:13:23,913 DEBUG [master/d1ba675678c9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-08T05:13:23,924 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46753-0x10118dcefab0001, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-08T05:13:23,924 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41485-0x10118dcefab0002, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-08T05:13:23,924 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39291-0x10118dcefab0003, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-08T05:13:23,924 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46753-0x10118dcefab0001, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T05:13:23,925 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41485-0x10118dcefab0002, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T05:13:23,925 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39291-0x10118dcefab0003, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T05:13:23,925 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37587-0x10118dcefab0000, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-08T05:13:23,925 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37587-0x10118dcefab0000, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T05:13:23,930 INFO [master/d1ba675678c9:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=d1ba675678c9,37587,1731042801796, sessionid=0x10118dcefab0000, setting cluster-up flag (Was=false) 2024-11-08T05:13:23,962 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39291-0x10118dcefab0003, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T05:13:23,962 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37587-0x10118dcefab0000, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T05:13:23,963 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41485-0x10118dcefab0002, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T05:13:23,963 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46753-0x10118dcefab0001, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T05:13:23,998 DEBUG [master/d1ba675678c9:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-08T05:13:24,001 DEBUG [master/d1ba675678c9:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=d1ba675678c9,37587,1731042801796 2024-11-08T05:13:24,026 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37587-0x10118dcefab0000, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T05:13:24,026 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46753-0x10118dcefab0001, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T05:13:24,026 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41485-0x10118dcefab0002, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T05:13:24,026 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39291-0x10118dcefab0003, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T05:13:24,061 DEBUG [master/d1ba675678c9:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-08T05:13:24,063 DEBUG [master/d1ba675678c9:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=d1ba675678c9,37587,1731042801796 2024-11-08T05:13:24,070 INFO [master/d1ba675678c9:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-08T05:13:24,135 DEBUG [master/d1ba675678c9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-08T05:13:24,144 INFO [master/d1ba675678c9:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-08T05:13:24,150 INFO [master/d1ba675678c9:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-08T05:13:24,156 INFO [RS:1;d1ba675678c9:41485 {}] regionserver.HRegionServer(746): ClusterId : 22414151-4211-4121-a8ec-5820592eaa4e 2024-11-08T05:13:24,156 INFO [RS:2;d1ba675678c9:39291 {}] regionserver.HRegionServer(746): ClusterId : 22414151-4211-4121-a8ec-5820592eaa4e 2024-11-08T05:13:24,156 INFO [RS:0;d1ba675678c9:46753 {}] regionserver.HRegionServer(746): ClusterId : 22414151-4211-4121-a8ec-5820592eaa4e 2024-11-08T05:13:24,155 DEBUG [master/d1ba675678c9:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: d1ba675678c9,37587,1731042801796 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-08T05:13:24,159 DEBUG [RS:0;d1ba675678c9:46753 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-08T05:13:24,159 DEBUG [RS:1;d1ba675678c9:41485 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-08T05:13:24,159 DEBUG [RS:2;d1ba675678c9:39291 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-08T05:13:24,168 DEBUG [RS:2;d1ba675678c9:39291 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-08T05:13:24,168 DEBUG [RS:1;d1ba675678c9:41485 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-08T05:13:24,172 DEBUG [RS:2;d1ba675678c9:39291 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-08T05:13:24,172 DEBUG [RS:0;d1ba675678c9:46753 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-08T05:13:24,172 DEBUG [master/d1ba675678c9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/d1ba675678c9:0, corePoolSize=5, maxPoolSize=5 2024-11-08T05:13:24,172 DEBUG [RS:1;d1ba675678c9:41485 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-08T05:13:24,172 DEBUG [RS:0;d1ba675678c9:46753 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-08T05:13:24,173 DEBUG [master/d1ba675678c9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/d1ba675678c9:0, corePoolSize=5, maxPoolSize=5 2024-11-08T05:13:24,173 DEBUG [master/d1ba675678c9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/d1ba675678c9:0, corePoolSize=5, maxPoolSize=5 2024-11-08T05:13:24,173 DEBUG [master/d1ba675678c9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/d1ba675678c9:0, corePoolSize=5, maxPoolSize=5 2024-11-08T05:13:24,173 DEBUG [master/d1ba675678c9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/d1ba675678c9:0, corePoolSize=10, maxPoolSize=10 2024-11-08T05:13:24,173 DEBUG [master/d1ba675678c9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/d1ba675678c9:0, corePoolSize=1, maxPoolSize=1 2024-11-08T05:13:24,174 DEBUG [master/d1ba675678c9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/d1ba675678c9:0, corePoolSize=2, maxPoolSize=2 2024-11-08T05:13:24,174 DEBUG [master/d1ba675678c9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/d1ba675678c9:0, corePoolSize=1, maxPoolSize=1 2024-11-08T05:13:24,178 INFO [master/d1ba675678c9:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731042834178 2024-11-08T05:13:24,180 INFO [master/d1ba675678c9:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-08T05:13:24,185 INFO [master/d1ba675678c9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-08T05:13:24,186 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-08T05:13:24,186 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-08T05:13:24,189 INFO [master/d1ba675678c9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-08T05:13:24,190 INFO [master/d1ba675678c9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-08T05:13:24,190 INFO [master/d1ba675678c9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-08T05:13:24,190 INFO [master/d1ba675678c9:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-08T05:13:24,191 INFO [master/d1ba675678c9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-08T05:13:24,193 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T05:13:24,195 INFO [master/d1ba675678c9:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-08T05:13:24,193 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-08T05:13:24,195 DEBUG [RS:2;d1ba675678c9:39291 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-08T05:13:24,196 DEBUG [RS:2;d1ba675678c9:39291 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53bba6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d1ba675678c9/172.17.0.3:0 2024-11-08T05:13:24,196 INFO [master/d1ba675678c9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-08T05:13:24,197 INFO [master/d1ba675678c9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-08T05:13:24,199 INFO [master/d1ba675678c9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-08T05:13:24,199 DEBUG [RS:1;d1ba675678c9:41485 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-08T05:13:24,199 INFO [master/d1ba675678c9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-08T05:13:24,201 DEBUG [RS:1;d1ba675678c9:41485 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21786275, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d1ba675678c9/172.17.0.3:0 2024-11-08T05:13:24,204 DEBUG [master/d1ba675678c9:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/d1ba675678c9:0:becomeActiveMaster-HFileCleaner.large.0-1731042804201,5,FailOnTimeoutGroup] 2024-11-08T05:13:24,206 DEBUG [RS:0;d1ba675678c9:46753 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-08T05:13:24,207 DEBUG [RS:0;d1ba675678c9:46753 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@422351a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d1ba675678c9/172.17.0.3:0 2024-11-08T05:13:24,216 DEBUG [RS:2;d1ba675678c9:39291 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;d1ba675678c9:39291 2024-11-08T05:13:24,220 INFO [RS:2;d1ba675678c9:39291 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-08T05:13:24,220 INFO [RS:2;d1ba675678c9:39291 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-08T05:13:24,220 DEBUG [RS:2;d1ba675678c9:39291 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-08T05:13:24,224 DEBUG [RS:1;d1ba675678c9:41485 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;d1ba675678c9:41485 2024-11-08T05:13:24,224 DEBUG [master/d1ba675678c9:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/d1ba675678c9:0:becomeActiveMaster-HFileCleaner.small.0-1731042804205,5,FailOnTimeoutGroup] 2024-11-08T05:13:24,224 INFO [RS:1;d1ba675678c9:41485 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-08T05:13:24,224 INFO [RS:2;d1ba675678c9:39291 {}] regionserver.HRegionServer(2659): reportForDuty to master=d1ba675678c9,37587,1731042801796 with port=39291, startcode=1731042803020 2024-11-08T05:13:24,224 INFO [master/d1ba675678c9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-08T05:13:24,224 INFO [RS:1;d1ba675678c9:41485 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-08T05:13:24,225 INFO [master/d1ba675678c9:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-08T05:13:24,225 DEBUG [RS:1;d1ba675678c9:41485 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-08T05:13:24,226 INFO [RS:1;d1ba675678c9:41485 {}] regionserver.HRegionServer(2659): reportForDuty to master=d1ba675678c9,37587,1731042801796 with port=41485, startcode=1731042802970 2024-11-08T05:13:24,226 INFO [master/d1ba675678c9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-08T05:13:24,226 INFO [master/d1ba675678c9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-08T05:13:24,228 DEBUG [RS:0;d1ba675678c9:46753 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;d1ba675678c9:46753 2024-11-08T05:13:24,228 INFO [RS:0;d1ba675678c9:46753 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-08T05:13:24,228 INFO [RS:0;d1ba675678c9:46753 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-08T05:13:24,228 DEBUG [RS:0;d1ba675678c9:46753 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-08T05:13:24,229 WARN [IPC Server handler 3 on default port 45429 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-08T05:13:24,230 WARN [IPC Server handler 3 on default port 45429 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-08T05:13:24,230 WARN [IPC Server handler 3 on default port 45429 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-08T05:13:24,232 INFO [RS:0;d1ba675678c9:46753 {}] regionserver.HRegionServer(2659): reportForDuty to master=d1ba675678c9,37587,1731042801796 with port=46753, startcode=1731042802851 2024-11-08T05:13:24,239 DEBUG [RS:2;d1ba675678c9:39291 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-08T05:13:24,239 DEBUG [RS:1;d1ba675678c9:41485 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-08T05:13:24,239 DEBUG [RS:0;d1ba675678c9:46753 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-08T05:13:24,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741831_1007 (size=1321) 2024-11-08T05:13:24,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741831_1007 (size=1321) 2024-11-08T05:13:24,245 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-08T05:13:24,245 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c 2024-11-08T05:13:24,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741832_1008 (size=32) 2024-11-08T05:13:24,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741832_1008 (size=32) 2024-11-08T05:13:24,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34565 is added to blk_1073741832_1008 (size=32) 2024-11-08T05:13:24,271 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T05:13:24,274 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-08T05:13:24,277 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-08T05:13:24,277 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T05:13:24,278 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T05:13:24,279 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-08T05:13:24,281 INFO [HMasterForTest-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60803, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-08T05:13:24,281 INFO [HMasterForTest-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56281, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-11-08T05:13:24,281 INFO [HMasterForTest-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39483, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-11-08T05:13:24,283 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-08T05:13:24,283 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T05:13:24,284 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T05:13:24,285 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-08T05:13:24,287 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-08T05:13:24,287 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T05:13:24,288 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T05:13:24,288 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-08T05:13:24,289 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37587 {}] master.ServerManager(363): Checking decommissioned status of RegionServer d1ba675678c9,46753,1731042802851 2024-11-08T05:13:24,291 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-08T05:13:24,291 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T05:13:24,292 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37587 {}] master.ServerManager(517): Registering regionserver=d1ba675678c9,46753,1731042802851 2024-11-08T05:13:24,292 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T05:13:24,293 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-08T05:13:24,294 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/hbase/meta/1588230740 2024-11-08T05:13:24,296 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/hbase/meta/1588230740 2024-11-08T05:13:24,300 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-08T05:13:24,300 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-08T05:13:24,302 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-08T05:13:24,305 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-08T05:13:24,310 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T05:13:24,310 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37587 {}] master.ServerManager(363): Checking decommissioned status of RegionServer d1ba675678c9,41485,1731042802970 2024-11-08T05:13:24,310 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37587 {}] master.ServerManager(517): Registering regionserver=d1ba675678c9,41485,1731042802970 2024-11-08T05:13:24,312 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65733235, jitterRate=-0.020498469471931458}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-08T05:13:24,313 DEBUG [RS:0;d1ba675678c9:46753 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c 2024-11-08T05:13:24,313 DEBUG [RS:0;d1ba675678c9:46753 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:45429 2024-11-08T05:13:24,313 DEBUG [RS:0;d1ba675678c9:46753 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-08T05:13:24,317 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731042804271Initializing all the Stores at 1731042804273 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731042804273Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731042804273Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731042804273Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731042804273Cleaning up temporary data from old regions at 1731042804300 (+27 ms)Region opened successfully at 1731042804317 (+17 ms) 2024-11-08T05:13:24,317 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37587 {}] master.ServerManager(363): Checking decommissioned status of RegionServer d1ba675678c9,39291,1731042803020 2024-11-08T05:13:24,317 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-08T05:13:24,317 DEBUG [RS:1;d1ba675678c9:41485 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c 2024-11-08T05:13:24,317 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37587 {}] master.ServerManager(517): Registering regionserver=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:24,317 DEBUG [RS:1;d1ba675678c9:41485 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:45429 2024-11-08T05:13:24,317 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-08T05:13:24,317 DEBUG [RS:1;d1ba675678c9:41485 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-08T05:13:24,317 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-08T05:13:24,317 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-08T05:13:24,317 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-08T05:13:24,319 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-08T05:13:24,319 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731042804317Disabling compacts and flushes for region at 1731042804317Disabling writes for close at 1731042804317Writing region close event to WAL at 1731042804318 (+1 ms)Closed at 1731042804319 (+1 ms) 2024-11-08T05:13:24,320 DEBUG [RS:2;d1ba675678c9:39291 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c 2024-11-08T05:13:24,320 DEBUG [RS:2;d1ba675678c9:39291 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:45429 2024-11-08T05:13:24,320 DEBUG [RS:2;d1ba675678c9:39291 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-08T05:13:24,321 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-08T05:13:24,321 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-08T05:13:24,324 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37587-0x10118dcefab0000, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-08T05:13:24,328 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-08T05:13:24,338 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T05:13:24,341 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-08T05:13:24,363 DEBUG [RS:0;d1ba675678c9:46753 {}] zookeeper.ZKUtil(111): regionserver:46753-0x10118dcefab0001, quorum=127.0.0.1:57355, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/d1ba675678c9,46753,1731042802851 2024-11-08T05:13:24,363 WARN [RS:0;d1ba675678c9:46753 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-08T05:13:24,363 DEBUG [RS:2;d1ba675678c9:39291 {}] zookeeper.ZKUtil(111): regionserver:39291-0x10118dcefab0003, quorum=127.0.0.1:57355, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/d1ba675678c9,39291,1731042803020 2024-11-08T05:13:24,363 INFO [RS:0;d1ba675678c9:46753 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-08T05:13:24,363 DEBUG [RS:1;d1ba675678c9:41485 {}] zookeeper.ZKUtil(111): regionserver:41485-0x10118dcefab0002, quorum=127.0.0.1:57355, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/d1ba675678c9,41485,1731042802970 2024-11-08T05:13:24,363 WARN [RS:2;d1ba675678c9:39291 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-08T05:13:24,363 WARN [RS:1;d1ba675678c9:41485 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-08T05:13:24,363 DEBUG [RS:0;d1ba675678c9:46753 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/WALs/d1ba675678c9,46753,1731042802851 2024-11-08T05:13:24,363 INFO [RS:2;d1ba675678c9:39291 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-08T05:13:24,363 INFO [RS:1;d1ba675678c9:41485 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-08T05:13:24,363 DEBUG [RS:1;d1ba675678c9:41485 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/WALs/d1ba675678c9,41485,1731042802970 2024-11-08T05:13:24,363 DEBUG [RS:2;d1ba675678c9:39291 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/WALs/d1ba675678c9,39291,1731042803020 2024-11-08T05:13:24,365 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [d1ba675678c9,46753,1731042802851] 2024-11-08T05:13:24,365 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [d1ba675678c9,41485,1731042802970] 2024-11-08T05:13:24,365 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [d1ba675678c9,39291,1731042803020] 2024-11-08T05:13:24,391 INFO [RS:1;d1ba675678c9:41485 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-08T05:13:24,391 INFO [RS:0;d1ba675678c9:46753 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-08T05:13:24,391 INFO [RS:2;d1ba675678c9:39291 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-08T05:13:24,406 INFO [RS:2;d1ba675678c9:39291 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-08T05:13:24,406 INFO [RS:1;d1ba675678c9:41485 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-08T05:13:24,406 INFO [RS:0;d1ba675678c9:46753 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-08T05:13:24,411 INFO [RS:0;d1ba675678c9:46753 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-08T05:13:24,411 INFO [RS:1;d1ba675678c9:41485 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-08T05:13:24,411 INFO [RS:2;d1ba675678c9:39291 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-08T05:13:24,411 INFO [RS:0;d1ba675678c9:46753 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T05:13:24,411 INFO [RS:1;d1ba675678c9:41485 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T05:13:24,411 INFO [RS:2;d1ba675678c9:39291 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T05:13:24,412 INFO [RS:2;d1ba675678c9:39291 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-08T05:13:24,412 INFO [RS:0;d1ba675678c9:46753 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-08T05:13:24,412 INFO [RS:1;d1ba675678c9:41485 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-08T05:13:24,418 INFO [RS:0;d1ba675678c9:46753 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-08T05:13:24,418 INFO [RS:1;d1ba675678c9:41485 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-08T05:13:24,418 INFO [RS:2;d1ba675678c9:39291 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-08T05:13:24,420 INFO [RS:0;d1ba675678c9:46753 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-08T05:13:24,420 INFO [RS:2;d1ba675678c9:39291 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-08T05:13:24,420 INFO [RS:1;d1ba675678c9:41485 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-08T05:13:24,420 DEBUG [RS:2;d1ba675678c9:39291 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/d1ba675678c9:0, corePoolSize=1, maxPoolSize=1 2024-11-08T05:13:24,420 DEBUG [RS:1;d1ba675678c9:41485 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/d1ba675678c9:0, corePoolSize=1, maxPoolSize=1 2024-11-08T05:13:24,420 DEBUG [RS:0;d1ba675678c9:46753 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/d1ba675678c9:0, corePoolSize=1, maxPoolSize=1 2024-11-08T05:13:24,421 DEBUG [RS:2;d1ba675678c9:39291 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/d1ba675678c9:0, corePoolSize=1, maxPoolSize=1 2024-11-08T05:13:24,421 DEBUG [RS:1;d1ba675678c9:41485 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/d1ba675678c9:0, corePoolSize=1, maxPoolSize=1 2024-11-08T05:13:24,421 DEBUG [RS:0;d1ba675678c9:46753 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/d1ba675678c9:0, corePoolSize=1, maxPoolSize=1 2024-11-08T05:13:24,421 DEBUG [RS:2;d1ba675678c9:39291 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/d1ba675678c9:0, corePoolSize=1, maxPoolSize=1 2024-11-08T05:13:24,421 DEBUG [RS:1;d1ba675678c9:41485 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/d1ba675678c9:0, corePoolSize=1, maxPoolSize=1 2024-11-08T05:13:24,421 DEBUG [RS:0;d1ba675678c9:46753 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/d1ba675678c9:0, corePoolSize=1, maxPoolSize=1 2024-11-08T05:13:24,421 DEBUG [RS:2;d1ba675678c9:39291 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/d1ba675678c9:0, corePoolSize=1, maxPoolSize=1 2024-11-08T05:13:24,421 DEBUG [RS:1;d1ba675678c9:41485 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/d1ba675678c9:0, corePoolSize=1, maxPoolSize=1 2024-11-08T05:13:24,421 DEBUG [RS:2;d1ba675678c9:39291 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/d1ba675678c9:0, corePoolSize=1, maxPoolSize=1 2024-11-08T05:13:24,421 DEBUG [RS:0;d1ba675678c9:46753 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/d1ba675678c9:0, corePoolSize=1, maxPoolSize=1 2024-11-08T05:13:24,421 DEBUG [RS:1;d1ba675678c9:41485 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/d1ba675678c9:0, corePoolSize=1, maxPoolSize=1 2024-11-08T05:13:24,421 DEBUG [RS:2;d1ba675678c9:39291 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/d1ba675678c9:0, corePoolSize=2, maxPoolSize=2 2024-11-08T05:13:24,421 DEBUG [RS:0;d1ba675678c9:46753 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/d1ba675678c9:0, corePoolSize=1, maxPoolSize=1 2024-11-08T05:13:24,421 DEBUG [RS:1;d1ba675678c9:41485 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/d1ba675678c9:0, corePoolSize=2, maxPoolSize=2 2024-11-08T05:13:24,421 DEBUG [RS:2;d1ba675678c9:39291 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/d1ba675678c9:0, corePoolSize=1, maxPoolSize=1 2024-11-08T05:13:24,421 DEBUG [RS:0;d1ba675678c9:46753 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/d1ba675678c9:0, corePoolSize=2, maxPoolSize=2 2024-11-08T05:13:24,421 DEBUG [RS:1;d1ba675678c9:41485 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/d1ba675678c9:0, corePoolSize=1, maxPoolSize=1 2024-11-08T05:13:24,421 DEBUG [RS:2;d1ba675678c9:39291 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/d1ba675678c9:0, corePoolSize=1, maxPoolSize=1 2024-11-08T05:13:24,422 DEBUG [RS:2;d1ba675678c9:39291 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/d1ba675678c9:0, corePoolSize=1, maxPoolSize=1 2024-11-08T05:13:24,422 DEBUG [RS:0;d1ba675678c9:46753 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/d1ba675678c9:0, corePoolSize=1, maxPoolSize=1 2024-11-08T05:13:24,422 DEBUG [RS:1;d1ba675678c9:41485 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/d1ba675678c9:0, corePoolSize=1, maxPoolSize=1 2024-11-08T05:13:24,422 DEBUG [RS:2;d1ba675678c9:39291 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/d1ba675678c9:0, corePoolSize=1, maxPoolSize=1 2024-11-08T05:13:24,422 DEBUG [RS:0;d1ba675678c9:46753 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/d1ba675678c9:0, corePoolSize=1, maxPoolSize=1 2024-11-08T05:13:24,422 DEBUG [RS:1;d1ba675678c9:41485 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/d1ba675678c9:0, corePoolSize=1, maxPoolSize=1 2024-11-08T05:13:24,422 DEBUG [RS:2;d1ba675678c9:39291 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/d1ba675678c9:0, corePoolSize=1, maxPoolSize=1 2024-11-08T05:13:24,422 DEBUG [RS:2;d1ba675678c9:39291 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/d1ba675678c9:0, corePoolSize=1, maxPoolSize=1 2024-11-08T05:13:24,422 DEBUG [RS:1;d1ba675678c9:41485 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/d1ba675678c9:0, corePoolSize=1, maxPoolSize=1 2024-11-08T05:13:24,422 DEBUG [RS:0;d1ba675678c9:46753 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/d1ba675678c9:0, corePoolSize=1, maxPoolSize=1 2024-11-08T05:13:24,422 DEBUG [RS:2;d1ba675678c9:39291 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/d1ba675678c9:0, corePoolSize=3, maxPoolSize=3 2024-11-08T05:13:24,422 DEBUG [RS:1;d1ba675678c9:41485 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/d1ba675678c9:0, corePoolSize=1, maxPoolSize=1 2024-11-08T05:13:24,422 DEBUG [RS:2;d1ba675678c9:39291 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/d1ba675678c9:0, corePoolSize=3, maxPoolSize=3 2024-11-08T05:13:24,422 DEBUG [RS:0;d1ba675678c9:46753 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/d1ba675678c9:0, corePoolSize=1, maxPoolSize=1 2024-11-08T05:13:24,422 DEBUG [RS:1;d1ba675678c9:41485 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/d1ba675678c9:0, corePoolSize=1, maxPoolSize=1 2024-11-08T05:13:24,422 DEBUG [RS:0;d1ba675678c9:46753 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/d1ba675678c9:0, corePoolSize=1, maxPoolSize=1 2024-11-08T05:13:24,422 DEBUG [RS:1;d1ba675678c9:41485 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/d1ba675678c9:0, corePoolSize=3, maxPoolSize=3 2024-11-08T05:13:24,422 DEBUG [RS:0;d1ba675678c9:46753 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/d1ba675678c9:0, corePoolSize=1, maxPoolSize=1 2024-11-08T05:13:24,422 DEBUG [RS:1;d1ba675678c9:41485 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/d1ba675678c9:0, corePoolSize=3, maxPoolSize=3 2024-11-08T05:13:24,423 DEBUG [RS:0;d1ba675678c9:46753 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/d1ba675678c9:0, corePoolSize=3, maxPoolSize=3 2024-11-08T05:13:24,423 DEBUG [RS:0;d1ba675678c9:46753 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/d1ba675678c9:0, corePoolSize=3, maxPoolSize=3 2024-11-08T05:13:24,424 INFO [RS:2;d1ba675678c9:39291 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-08T05:13:24,424 INFO [RS:1;d1ba675678c9:41485 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-08T05:13:24,424 INFO [RS:2;d1ba675678c9:39291 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-08T05:13:24,424 INFO [RS:1;d1ba675678c9:41485 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-08T05:13:24,424 INFO [RS:2;d1ba675678c9:39291 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T05:13:24,424 INFO [RS:1;d1ba675678c9:41485 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T05:13:24,424 INFO [RS:2;d1ba675678c9:39291 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-08T05:13:24,424 INFO [RS:1;d1ba675678c9:41485 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-08T05:13:24,424 INFO [RS:2;d1ba675678c9:39291 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-08T05:13:24,424 INFO [RS:1;d1ba675678c9:41485 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-08T05:13:24,425 INFO [RS:1;d1ba675678c9:41485 {}] hbase.ChoreService(168): Chore ScheduledChore name=d1ba675678c9,41485,1731042802970-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-08T05:13:24,425 INFO [RS:2;d1ba675678c9:39291 {}] hbase.ChoreService(168): Chore ScheduledChore name=d1ba675678c9,39291,1731042803020-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-08T05:13:24,427 INFO [RS:0;d1ba675678c9:46753 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-08T05:13:24,427 INFO [RS:0;d1ba675678c9:46753 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-08T05:13:24,427 INFO [RS:0;d1ba675678c9:46753 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T05:13:24,427 INFO [RS:0;d1ba675678c9:46753 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-08T05:13:24,427 INFO [RS:0;d1ba675678c9:46753 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-08T05:13:24,427 INFO [RS:0;d1ba675678c9:46753 {}] hbase.ChoreService(168): Chore ScheduledChore name=d1ba675678c9,46753,1731042802851-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-08T05:13:24,453 INFO [RS:1;d1ba675678c9:41485 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-08T05:13:24,453 INFO [RS:2;d1ba675678c9:39291 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-08T05:13:24,453 INFO [RS:0;d1ba675678c9:46753 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-08T05:13:24,455 INFO [RS:0;d1ba675678c9:46753 {}] hbase.ChoreService(168): Chore ScheduledChore name=d1ba675678c9,46753,1731042802851-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T05:13:24,455 INFO [RS:2;d1ba675678c9:39291 {}] hbase.ChoreService(168): Chore ScheduledChore name=d1ba675678c9,39291,1731042803020-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T05:13:24,455 INFO [RS:1;d1ba675678c9:41485 {}] hbase.ChoreService(168): Chore ScheduledChore name=d1ba675678c9,41485,1731042802970-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T05:13:24,455 INFO [RS:1;d1ba675678c9:41485 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T05:13:24,455 INFO [RS:2;d1ba675678c9:39291 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T05:13:24,455 INFO [RS:0;d1ba675678c9:46753 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T05:13:24,456 INFO [RS:1;d1ba675678c9:41485 {}] regionserver.Replication(171): d1ba675678c9,41485,1731042802970 started 2024-11-08T05:13:24,456 INFO [RS:2;d1ba675678c9:39291 {}] regionserver.Replication(171): d1ba675678c9,39291,1731042803020 started 2024-11-08T05:13:24,456 INFO [RS:0;d1ba675678c9:46753 {}] regionserver.Replication(171): d1ba675678c9,46753,1731042802851 started 2024-11-08T05:13:24,483 INFO [RS:1;d1ba675678c9:41485 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T05:13:24,483 INFO [RS:0;d1ba675678c9:46753 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T05:13:24,483 INFO [RS:2;d1ba675678c9:39291 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T05:13:24,483 INFO [RS:2;d1ba675678c9:39291 {}] regionserver.HRegionServer(1482): Serving as d1ba675678c9,39291,1731042803020, RpcServer on d1ba675678c9/172.17.0.3:39291, sessionid=0x10118dcefab0003 2024-11-08T05:13:24,483 INFO [RS:1;d1ba675678c9:41485 {}] regionserver.HRegionServer(1482): Serving as d1ba675678c9,41485,1731042802970, RpcServer on d1ba675678c9/172.17.0.3:41485, sessionid=0x10118dcefab0002 2024-11-08T05:13:24,483 INFO [RS:0;d1ba675678c9:46753 {}] regionserver.HRegionServer(1482): Serving as d1ba675678c9,46753,1731042802851, RpcServer on d1ba675678c9/172.17.0.3:46753, sessionid=0x10118dcefab0001 2024-11-08T05:13:24,484 DEBUG [RS:1;d1ba675678c9:41485 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-08T05:13:24,484 DEBUG [RS:0;d1ba675678c9:46753 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-08T05:13:24,484 DEBUG [RS:2;d1ba675678c9:39291 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-08T05:13:24,484 DEBUG [RS:2;d1ba675678c9:39291 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager d1ba675678c9,39291,1731042803020 2024-11-08T05:13:24,484 DEBUG [RS:1;d1ba675678c9:41485 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager d1ba675678c9,41485,1731042802970 2024-11-08T05:13:24,484 DEBUG [RS:0;d1ba675678c9:46753 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager d1ba675678c9,46753,1731042802851 2024-11-08T05:13:24,485 DEBUG [RS:2;d1ba675678c9:39291 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd1ba675678c9,39291,1731042803020' 2024-11-08T05:13:24,485 DEBUG [RS:1;d1ba675678c9:41485 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd1ba675678c9,41485,1731042802970' 2024-11-08T05:13:24,485 DEBUG [RS:0;d1ba675678c9:46753 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd1ba675678c9,46753,1731042802851' 2024-11-08T05:13:24,485 DEBUG [RS:2;d1ba675678c9:39291 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-08T05:13:24,485 DEBUG [RS:1;d1ba675678c9:41485 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-08T05:13:24,485 DEBUG [RS:0;d1ba675678c9:46753 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-08T05:13:24,486 DEBUG [RS:0;d1ba675678c9:46753 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-08T05:13:24,486 DEBUG [RS:2;d1ba675678c9:39291 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-08T05:13:24,486 DEBUG [RS:1;d1ba675678c9:41485 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-08T05:13:24,486 DEBUG [RS:2;d1ba675678c9:39291 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-08T05:13:24,486 DEBUG [RS:2;d1ba675678c9:39291 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-08T05:13:24,486 DEBUG [RS:0;d1ba675678c9:46753 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-08T05:13:24,486 DEBUG [RS:1;d1ba675678c9:41485 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-08T05:13:24,486 DEBUG [RS:0;d1ba675678c9:46753 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-08T05:13:24,486 DEBUG [RS:1;d1ba675678c9:41485 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-08T05:13:24,487 DEBUG [RS:1;d1ba675678c9:41485 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager d1ba675678c9,41485,1731042802970 2024-11-08T05:13:24,487 DEBUG [RS:2;d1ba675678c9:39291 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager d1ba675678c9,39291,1731042803020 2024-11-08T05:13:24,487 DEBUG [RS:0;d1ba675678c9:46753 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager d1ba675678c9,46753,1731042802851 2024-11-08T05:13:24,487 DEBUG [RS:1;d1ba675678c9:41485 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd1ba675678c9,41485,1731042802970' 2024-11-08T05:13:24,487 DEBUG [RS:2;d1ba675678c9:39291 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd1ba675678c9,39291,1731042803020' 2024-11-08T05:13:24,487 DEBUG [RS:0;d1ba675678c9:46753 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd1ba675678c9,46753,1731042802851' 2024-11-08T05:13:24,487 DEBUG [RS:1;d1ba675678c9:41485 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-08T05:13:24,487 DEBUG [RS:2;d1ba675678c9:39291 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-08T05:13:24,487 DEBUG [RS:0;d1ba675678c9:46753 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-08T05:13:24,487 DEBUG [RS:1;d1ba675678c9:41485 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-08T05:13:24,487 DEBUG [RS:2;d1ba675678c9:39291 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-08T05:13:24,487 DEBUG [RS:0;d1ba675678c9:46753 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-08T05:13:24,488 DEBUG [RS:2;d1ba675678c9:39291 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-08T05:13:24,488 DEBUG [RS:1;d1ba675678c9:41485 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-08T05:13:24,488 INFO [RS:2;d1ba675678c9:39291 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-08T05:13:24,488 DEBUG [RS:0;d1ba675678c9:46753 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-08T05:13:24,488 INFO [RS:1;d1ba675678c9:41485 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-08T05:13:24,488 INFO [RS:1;d1ba675678c9:41485 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-08T05:13:24,488 INFO [RS:0;d1ba675678c9:46753 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-08T05:13:24,488 INFO [RS:2;d1ba675678c9:39291 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-08T05:13:24,488 INFO [RS:0;d1ba675678c9:46753 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-08T05:13:24,493 WARN [d1ba675678c9:37587 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-08T05:13:24,596 INFO [RS:1;d1ba675678c9:41485 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-08T05:13:24,596 INFO [RS:2;d1ba675678c9:39291 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-08T05:13:24,596 INFO [RS:0;d1ba675678c9:46753 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-08T05:13:24,599 INFO [RS:2;d1ba675678c9:39291 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d1ba675678c9%2C39291%2C1731042803020, suffix=, logDir=hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/WALs/d1ba675678c9,39291,1731042803020, archiveDir=hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/oldWALs, maxLogs=32 2024-11-08T05:13:24,599 INFO [RS:1;d1ba675678c9:41485 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d1ba675678c9%2C41485%2C1731042802970, suffix=, logDir=hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/WALs/d1ba675678c9,41485,1731042802970, archiveDir=hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/oldWALs, maxLogs=32 2024-11-08T05:13:24,599 INFO [RS:0;d1ba675678c9:46753 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d1ba675678c9%2C46753%2C1731042802851, suffix=, logDir=hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/WALs/d1ba675678c9,46753,1731042802851, archiveDir=hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/oldWALs, maxLogs=32 2024-11-08T05:13:24,618 DEBUG [RS:0;d1ba675678c9:46753 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/WALs/d1ba675678c9,46753,1731042802851/d1ba675678c9%2C46753%2C1731042802851.1731042804603, exclude list is [], retry=0 2024-11-08T05:13:24,620 DEBUG [RS:1;d1ba675678c9:41485 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/WALs/d1ba675678c9,41485,1731042802970/d1ba675678c9%2C41485%2C1731042802970.1731042804603, exclude list is [], retry=0 2024-11-08T05:13:24,620 DEBUG [RS:2;d1ba675678c9:39291 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/WALs/d1ba675678c9,39291,1731042803020/d1ba675678c9%2C39291%2C1731042803020.1731042804603, exclude list is [], retry=0 2024-11-08T05:13:24,620 WARN [IPC Server handler 4 on default port 45429 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-08T05:13:24,620 WARN [IPC Server handler 4 on default port 45429 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-08T05:13:24,620 WARN [IPC Server handler 4 on default port 45429 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-08T05:13:24,622 WARN [IPC Server handler 1 on default port 45429 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-08T05:13:24,622 WARN [IPC Server handler 1 on default port 45429 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-08T05:13:24,622 WARN [IPC Server handler 1 on default port 45429 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-08T05:13:24,623 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41203,DS-097c58aa-51b2-4f6f-9fb5-10702640c273,DISK] 2024-11-08T05:13:24,623 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37041,DS-7382e2ea-12bb-4310-9c38-6f88a0ed4f48,DISK] 2024-11-08T05:13:24,626 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34565,DS-580c23be-6f59-4a94-8328-84ad55cf144b,DISK] 2024-11-08T05:13:24,626 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37041,DS-7382e2ea-12bb-4310-9c38-6f88a0ed4f48,DISK] 2024-11-08T05:13:24,626 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41203,DS-097c58aa-51b2-4f6f-9fb5-10702640c273,DISK] 2024-11-08T05:13:24,626 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37041,DS-7382e2ea-12bb-4310-9c38-6f88a0ed4f48,DISK] 2024-11-08T05:13:24,649 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41203,DS-097c58aa-51b2-4f6f-9fb5-10702640c273,DISK] 2024-11-08T05:13:24,655 INFO [RS:2;d1ba675678c9:39291 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/WALs/d1ba675678c9,39291,1731042803020/d1ba675678c9%2C39291%2C1731042803020.1731042804603 2024-11-08T05:13:24,655 INFO [RS:1;d1ba675678c9:41485 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/WALs/d1ba675678c9,41485,1731042802970/d1ba675678c9%2C41485%2C1731042802970.1731042804603 2024-11-08T05:13:24,655 INFO [RS:0;d1ba675678c9:46753 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/WALs/d1ba675678c9,46753,1731042802851/d1ba675678c9%2C46753%2C1731042802851.1731042804603 2024-11-08T05:13:24,656 DEBUG [RS:1;d1ba675678c9:41485 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34771:34771),(127.0.0.1/127.0.0.1:35281:35281),(127.0.0.1/127.0.0.1:43885:43885)] 2024-11-08T05:13:24,656 DEBUG [RS:2;d1ba675678c9:39291 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43885:43885),(127.0.0.1/127.0.0.1:35281:35281)] 2024-11-08T05:13:24,656 DEBUG [RS:0;d1ba675678c9:46753 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35281:35281),(127.0.0.1/127.0.0.1:43885:43885)] 2024-11-08T05:13:24,750 DEBUG [d1ba675678c9:37587 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-11-08T05:13:24,761 DEBUG [d1ba675678c9:37587 {}] balancer.BalancerClusterState(204): Hosts are {d1ba675678c9=0} racks are {/default-rack=0} 2024-11-08T05:13:24,767 DEBUG [d1ba675678c9:37587 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-08T05:13:24,767 DEBUG [d1ba675678c9:37587 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-08T05:13:24,767 DEBUG [d1ba675678c9:37587 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-08T05:13:24,767 DEBUG [d1ba675678c9:37587 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-08T05:13:24,767 DEBUG [d1ba675678c9:37587 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-08T05:13:24,767 DEBUG [d1ba675678c9:37587 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-08T05:13:24,767 INFO [d1ba675678c9:37587 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-08T05:13:24,767 INFO [d1ba675678c9:37587 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-08T05:13:24,767 INFO [d1ba675678c9:37587 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-08T05:13:24,767 DEBUG [d1ba675678c9:37587 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-08T05:13:24,776 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=d1ba675678c9,46753,1731042802851 2024-11-08T05:13:24,782 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d1ba675678c9,46753,1731042802851, state=OPENING 2024-11-08T05:13:24,815 DEBUG [PEWorker-1 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-08T05:13:24,903 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37587-0x10118dcefab0000, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T05:13:24,903 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41485-0x10118dcefab0002, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T05:13:24,903 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46753-0x10118dcefab0001, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T05:13:24,904 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39291-0x10118dcefab0003, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T05:13:24,906 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T05:13:24,906 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T05:13:24,906 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T05:13:24,906 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T05:13:24,909 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T05:13:24,913 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=d1ba675678c9,46753,1731042802851}] 2024-11-08T05:13:25,090 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-08T05:13:25,092 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33925, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-08T05:13:25,104 INFO [RS_OPEN_META-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-08T05:13:25,104 INFO [RS_OPEN_META-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-08T05:13:25,105 INFO [RS_OPEN_META-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-08T05:13:25,108 INFO [RS_OPEN_META-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d1ba675678c9%2C46753%2C1731042802851.meta, suffix=.meta, logDir=hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/WALs/d1ba675678c9,46753,1731042802851, archiveDir=hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/oldWALs, maxLogs=32 2024-11-08T05:13:25,128 DEBUG [RS_OPEN_META-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/WALs/d1ba675678c9,46753,1731042802851/d1ba675678c9%2C46753%2C1731042802851.meta.1731042805109.meta, exclude list is [], retry=0 2024-11-08T05:13:25,131 WARN [IPC Server handler 2 on default port 45429 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-08T05:13:25,131 WARN [IPC Server handler 2 on default port 45429 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-08T05:13:25,131 WARN [IPC Server handler 2 on default port 45429 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-08T05:13:25,133 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37041,DS-7382e2ea-12bb-4310-9c38-6f88a0ed4f48,DISK] 2024-11-08T05:13:25,133 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41203,DS-097c58aa-51b2-4f6f-9fb5-10702640c273,DISK] 2024-11-08T05:13:25,136 INFO [RS_OPEN_META-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/WALs/d1ba675678c9,46753,1731042802851/d1ba675678c9%2C46753%2C1731042802851.meta.1731042805109.meta 2024-11-08T05:13:25,136 DEBUG [RS_OPEN_META-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35281:35281),(127.0.0.1/127.0.0.1:43885:43885)] 2024-11-08T05:13:25,136 DEBUG [RS_OPEN_META-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-08T05:13:25,138 DEBUG [RS_OPEN_META-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-08T05:13:25,141 DEBUG [RS_OPEN_META-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-08T05:13:25,145 INFO [RS_OPEN_META-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-08T05:13:25,148 DEBUG [RS_OPEN_META-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-08T05:13:25,149 DEBUG [RS_OPEN_META-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T05:13:25,149 DEBUG [RS_OPEN_META-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-08T05:13:25,149 DEBUG [RS_OPEN_META-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-08T05:13:25,153 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-08T05:13:25,154 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-08T05:13:25,155 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T05:13:25,155 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T05:13:25,156 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-08T05:13:25,157 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-08T05:13:25,157 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T05:13:25,158 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T05:13:25,158 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-08T05:13:25,160 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-08T05:13:25,160 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T05:13:25,161 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T05:13:25,161 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-08T05:13:25,163 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-08T05:13:25,163 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T05:13:25,164 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T05:13:25,164 DEBUG [RS_OPEN_META-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-08T05:13:25,165 DEBUG [RS_OPEN_META-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/hbase/meta/1588230740 2024-11-08T05:13:25,168 DEBUG [RS_OPEN_META-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/hbase/meta/1588230740 2024-11-08T05:13:25,171 DEBUG [RS_OPEN_META-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-08T05:13:25,171 DEBUG [RS_OPEN_META-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-08T05:13:25,172 DEBUG [RS_OPEN_META-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-08T05:13:25,175 DEBUG [RS_OPEN_META-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-08T05:13:25,177 INFO [RS_OPEN_META-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72315928, jitterRate=0.07759130001068115}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-08T05:13:25,177 DEBUG [RS_OPEN_META-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-08T05:13:25,179 DEBUG [RS_OPEN_META-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731042805150Writing region info on filesystem at 1731042805150Initializing all the Stores at 1731042805152 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731042805152Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731042805152Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731042805152Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731042805152Cleaning up temporary data from old regions at 1731042805171 (+19 ms)Running coprocessor post-open hooks at 1731042805177 (+6 ms)Region opened successfully at 1731042805179 (+2 ms) 2024-11-08T05:13:25,185 INFO [RS_OPEN_META-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731042805081 2024-11-08T05:13:25,196 DEBUG [RS_OPEN_META-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-08T05:13:25,197 INFO [RS_OPEN_META-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-08T05:13:25,200 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=d1ba675678c9,46753,1731042802851 2024-11-08T05:13:25,202 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d1ba675678c9,46753,1731042802851, state=OPEN 2024-11-08T05:13:25,229 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46753-0x10118dcefab0001, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T05:13:25,229 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41485-0x10118dcefab0002, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T05:13:25,229 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39291-0x10118dcefab0003, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T05:13:25,229 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37587-0x10118dcefab0000, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T05:13:25,229 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T05:13:25,229 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T05:13:25,229 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T05:13:25,229 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T05:13:25,230 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=d1ba675678c9,46753,1731042802851 2024-11-08T05:13:25,236 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-08T05:13:25,237 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d1ba675678c9,46753,1731042802851 in 318 msec 2024-11-08T05:13:25,244 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-08T05:13:25,244 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 911 msec 2024-11-08T05:13:25,246 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-08T05:13:25,246 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-08T05:13:25,265 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T05:13:25,266 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d1ba675678c9,46753,1731042802851, seqNum=-1] 2024-11-08T05:13:25,292 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T05:13:25,295 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58053, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T05:13:25,319 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.2180 sec 2024-11-08T05:13:25,319 INFO [master/d1ba675678c9:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731042805319, completionTime=-1 2024-11-08T05:13:25,322 INFO [master/d1ba675678c9:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-11-08T05:13:25,322 DEBUG [master/d1ba675678c9:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-08T05:13:25,353 INFO [master/d1ba675678c9:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=3 2024-11-08T05:13:25,353 INFO [master/d1ba675678c9:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731042865353 2024-11-08T05:13:25,353 INFO [master/d1ba675678c9:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731042925353 2024-11-08T05:13:25,353 INFO [master/d1ba675678c9:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 31 msec 2024-11-08T05:13:25,355 DEBUG [master/d1ba675678c9:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-08T05:13:25,364 INFO [master/d1ba675678c9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d1ba675678c9,37587,1731042801796-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T05:13:25,364 INFO [master/d1ba675678c9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d1ba675678c9,37587,1731042801796-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T05:13:25,364 INFO [master/d1ba675678c9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d1ba675678c9,37587,1731042801796-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T05:13:25,366 INFO [master/d1ba675678c9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-d1ba675678c9:37587, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T05:13:25,366 INFO [master/d1ba675678c9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-08T05:13:25,367 INFO [master/d1ba675678c9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-08T05:13:25,373 DEBUG [master/d1ba675678c9:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-08T05:13:25,396 INFO [master/d1ba675678c9:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.270sec 2024-11-08T05:13:25,398 INFO [master/d1ba675678c9:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-08T05:13:25,400 INFO [master/d1ba675678c9:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-08T05:13:25,401 INFO [master/d1ba675678c9:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-08T05:13:25,401 INFO [master/d1ba675678c9:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-08T05:13:25,401 INFO [master/d1ba675678c9:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-08T05:13:25,402 INFO [master/d1ba675678c9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d1ba675678c9,37587,1731042801796-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-08T05:13:25,403 INFO [master/d1ba675678c9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d1ba675678c9,37587,1731042801796-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-08T05:13:25,408 DEBUG [master/d1ba675678c9:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-08T05:13:25,409 INFO [master/d1ba675678c9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-08T05:13:25,410 INFO [master/d1ba675678c9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d1ba675678c9,37587,1731042801796-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T05:13:25,472 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ff0a408, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T05:13:25,475 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-08T05:13:25,475 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-08T05:13:25,479 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d1ba675678c9,37587,-1 for getting cluster id 2024-11-08T05:13:25,482 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T05:13:25,489 DEBUG [HMasterForTest-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '22414151-4211-4121-a8ec-5820592eaa4e' 2024-11-08T05:13:25,492 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T05:13:25,492 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "22414151-4211-4121-a8ec-5820592eaa4e" 2024-11-08T05:13:25,492 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71c5fdfc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T05:13:25,492 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d1ba675678c9,37587,-1] 2024-11-08T05:13:25,496 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T05:13:25,499 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T05:13:25,500 INFO [HMasterForTest-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37040, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T05:13:25,503 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6238c979, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T05:13:25,504 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T05:13:25,513 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d1ba675678c9,46753,1731042802851, seqNum=-1] 2024-11-08T05:13:25,514 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T05:13:25,518 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47890, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T05:13:25,574 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=d1ba675678c9,37587,1731042801796 2024-11-08T05:13:25,578 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-08T05:13:25,583 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is d1ba675678c9,37587,1731042801796 2024-11-08T05:13:25,585 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4556a49d 2024-11-08T05:13:25,587 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-08T05:13:25,590 INFO [HMasterForTest-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37048, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-08T05:13:25,601 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37587 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'test', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T05:13:25,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37587 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=test 2024-11-08T05:13:25,612 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T05:13:25,613 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37587 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "test" procId is: 4 2024-11-08T05:13:25,614 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T05:13:25,616 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T05:13:25,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37587 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-08T05:13:25,624 WARN [IPC Server handler 1 on default port 45429 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-08T05:13:25,624 WARN [IPC Server handler 1 on default port 45429 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-08T05:13:25,624 WARN [IPC Server handler 1 on default port 45429 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-08T05:13:25,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741837_1013 (size=358) 2024-11-08T05:13:25,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741837_1013 (size=358) 2024-11-08T05:13:25,635 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 6240e38715889f3bb77bb89478c81a6c, NAME => 'test,,1731042805591.6240e38715889f3bb77bb89478c81a6c.', STARTKEY => '', ENDKEY => 'bbb'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c 2024-11-08T05:13:25,635 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 68bbd0353f97ee12228f7798b0b987ea, NAME => 'test,ccc,1731042805591.68bbd0353f97ee12228f7798b0b987ea.', STARTKEY => 'ccc', ENDKEY => 'ddd'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c 2024-11-08T05:13:25,636 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => eeaad0e7e7aa66f0b51c273c0aaf953c, NAME => 'test,bbb,1731042805591.eeaad0e7e7aa66f0b51c273c0aaf953c.', STARTKEY => 'bbb', ENDKEY => 'ccc'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c 2024-11-08T05:13:25,641 WARN [IPC Server handler 3 on default port 45429 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-08T05:13:25,641 WARN [IPC Server handler 3 on default port 45429 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-08T05:13:25,641 WARN [IPC Server handler 1 on default port 45429 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-08T05:13:25,641 WARN [IPC Server handler 3 on default port 45429 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-08T05:13:25,641 WARN [IPC Server handler 1 on default port 45429 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-08T05:13:25,641 WARN [IPC Server handler 1 on default port 45429 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-08T05:13:25,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741839_1015 (size=42) 2024-11-08T05:13:25,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741839_1015 (size=42) 2024-11-08T05:13:25,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741840_1016 (size=45) 2024-11-08T05:13:25,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741840_1016 (size=45) 2024-11-08T05:13:25,670 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ccc,1731042805591.68bbd0353f97ee12228f7798b0b987ea.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T05:13:25,671 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 68bbd0353f97ee12228f7798b0b987ea, disabling compactions & flushes 2024-11-08T05:13:25,671 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ccc,1731042805591.68bbd0353f97ee12228f7798b0b987ea. 2024-11-08T05:13:25,671 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ccc,1731042805591.68bbd0353f97ee12228f7798b0b987ea. 2024-11-08T05:13:25,671 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ccc,1731042805591.68bbd0353f97ee12228f7798b0b987ea. after waiting 0 ms 2024-11-08T05:13:25,671 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ccc,1731042805591.68bbd0353f97ee12228f7798b0b987ea. 2024-11-08T05:13:25,671 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ccc,1731042805591.68bbd0353f97ee12228f7798b0b987ea. 2024-11-08T05:13:25,671 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 68bbd0353f97ee12228f7798b0b987ea: Waiting for close lock at 1731042805671Disabling compacts and flushes for region at 1731042805671Disabling writes for close at 1731042805671Writing region close event to WAL at 1731042805671Closed at 1731042805671 2024-11-08T05:13:25,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741838_1014 (size=45) 2024-11-08T05:13:25,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741838_1014 (size=45) 2024-11-08T05:13:25,673 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 64ddf686fed30cdb23b2e874efec598d, NAME => 'test,ddd,1731042805591.64ddf686fed30cdb23b2e874efec598d.', STARTKEY => 'ddd', ENDKEY => 'eee'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c 2024-11-08T05:13:25,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34565 is added to blk_1073741838_1014 (size=45) 2024-11-08T05:13:25,677 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,bbb,1731042805591.eeaad0e7e7aa66f0b51c273c0aaf953c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T05:13:25,677 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing eeaad0e7e7aa66f0b51c273c0aaf953c, disabling compactions & flushes 2024-11-08T05:13:25,677 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,bbb,1731042805591.eeaad0e7e7aa66f0b51c273c0aaf953c. 2024-11-08T05:13:25,678 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,bbb,1731042805591.eeaad0e7e7aa66f0b51c273c0aaf953c. 2024-11-08T05:13:25,678 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,bbb,1731042805591.eeaad0e7e7aa66f0b51c273c0aaf953c. after waiting 0 ms 2024-11-08T05:13:25,678 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,bbb,1731042805591.eeaad0e7e7aa66f0b51c273c0aaf953c. 2024-11-08T05:13:25,678 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,bbb,1731042805591.eeaad0e7e7aa66f0b51c273c0aaf953c. 2024-11-08T05:13:25,678 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for eeaad0e7e7aa66f0b51c273c0aaf953c: Waiting for close lock at 1731042805677Disabling compacts and flushes for region at 1731042805677Disabling writes for close at 1731042805678 (+1 ms)Writing region close event to WAL at 1731042805678Closed at 1731042805678 2024-11-08T05:13:25,679 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => d7680a9929f15640ca61551f186f33c1, NAME => 'test,eee,1731042805591.d7680a9929f15640ca61551f186f33c1.', STARTKEY => 'eee', ENDKEY => 'fff'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c 2024-11-08T05:13:25,679 WARN [IPC Server handler 3 on default port 45429 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-08T05:13:25,679 WARN [IPC Server handler 3 on default port 45429 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-08T05:13:25,679 WARN [IPC Server handler 3 on default port 45429 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-08T05:13:25,684 WARN [IPC Server handler 2 on default port 45429 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-08T05:13:25,685 WARN [IPC Server handler 2 on default port 45429 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-08T05:13:25,685 WARN [IPC Server handler 2 on default port 45429 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-08T05:13:25,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741841_1017 (size=45) 2024-11-08T05:13:25,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741841_1017 (size=45) 2024-11-08T05:13:25,691 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ddd,1731042805591.64ddf686fed30cdb23b2e874efec598d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T05:13:25,691 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 64ddf686fed30cdb23b2e874efec598d, disabling compactions & flushes 2024-11-08T05:13:25,691 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ddd,1731042805591.64ddf686fed30cdb23b2e874efec598d. 2024-11-08T05:13:25,691 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ddd,1731042805591.64ddf686fed30cdb23b2e874efec598d. 2024-11-08T05:13:25,691 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ddd,1731042805591.64ddf686fed30cdb23b2e874efec598d. after waiting 0 ms 2024-11-08T05:13:25,691 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ddd,1731042805591.64ddf686fed30cdb23b2e874efec598d. 2024-11-08T05:13:25,691 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ddd,1731042805591.64ddf686fed30cdb23b2e874efec598d. 2024-11-08T05:13:25,691 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 64ddf686fed30cdb23b2e874efec598d: Waiting for close lock at 1731042805691Disabling compacts and flushes for region at 1731042805691Disabling writes for close at 1731042805691Writing region close event to WAL at 1731042805691Closed at 1731042805691 2024-11-08T05:13:25,692 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 65d5312dbd2f96bf3b16bdfd7504dff6, NAME => 'test,fff,1731042805591.65d5312dbd2f96bf3b16bdfd7504dff6.', STARTKEY => 'fff', ENDKEY => 'ggg'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c 2024-11-08T05:13:25,698 WARN [IPC Server handler 3 on default port 45429 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-08T05:13:25,698 WARN [IPC Server handler 3 on default port 45429 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-08T05:13:25,698 WARN [IPC Server handler 3 on default port 45429 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-08T05:13:25,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741842_1018 (size=45) 2024-11-08T05:13:25,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741842_1018 (size=45) 2024-11-08T05:13:25,700 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,eee,1731042805591.d7680a9929f15640ca61551f186f33c1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T05:13:25,700 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing d7680a9929f15640ca61551f186f33c1, disabling compactions & flushes 2024-11-08T05:13:25,700 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,eee,1731042805591.d7680a9929f15640ca61551f186f33c1. 2024-11-08T05:13:25,700 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,eee,1731042805591.d7680a9929f15640ca61551f186f33c1. 2024-11-08T05:13:25,700 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,eee,1731042805591.d7680a9929f15640ca61551f186f33c1. after waiting 0 ms 2024-11-08T05:13:25,700 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,eee,1731042805591.d7680a9929f15640ca61551f186f33c1. 2024-11-08T05:13:25,700 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,eee,1731042805591.d7680a9929f15640ca61551f186f33c1. 2024-11-08T05:13:25,700 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for d7680a9929f15640ca61551f186f33c1: Waiting for close lock at 1731042805700Disabling compacts and flushes for region at 1731042805700Disabling writes for close at 1731042805700Writing region close event to WAL at 1731042805700Closed at 1731042805700 2024-11-08T05:13:25,701 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 9b3274d80dcb7deed617c19ac1de4a68, NAME => 'test,ggg,1731042805591.9b3274d80dcb7deed617c19ac1de4a68.', STARTKEY => 'ggg', ENDKEY => 'hhh'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c 2024-11-08T05:13:25,705 WARN [IPC Server handler 4 on default port 45429 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-08T05:13:25,705 WARN [IPC Server handler 4 on default port 45429 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-08T05:13:25,705 WARN [IPC Server handler 4 on default port 45429 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-08T05:13:25,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741843_1019 (size=45) 2024-11-08T05:13:25,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741843_1019 (size=45) 2024-11-08T05:13:25,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741844_1020 (size=45) 2024-11-08T05:13:25,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741844_1020 (size=45) 2024-11-08T05:13:25,717 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,fff,1731042805591.65d5312dbd2f96bf3b16bdfd7504dff6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T05:13:25,717 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,ggg,1731042805591.9b3274d80dcb7deed617c19ac1de4a68.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T05:13:25,717 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 9b3274d80dcb7deed617c19ac1de4a68, disabling compactions & flushes 2024-11-08T05:13:25,717 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 65d5312dbd2f96bf3b16bdfd7504dff6, disabling compactions & flushes 2024-11-08T05:13:25,717 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,ggg,1731042805591.9b3274d80dcb7deed617c19ac1de4a68. 2024-11-08T05:13:25,717 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,fff,1731042805591.65d5312dbd2f96bf3b16bdfd7504dff6. 2024-11-08T05:13:25,717 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ggg,1731042805591.9b3274d80dcb7deed617c19ac1de4a68. 2024-11-08T05:13:25,717 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,fff,1731042805591.65d5312dbd2f96bf3b16bdfd7504dff6. 2024-11-08T05:13:25,717 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,ggg,1731042805591.9b3274d80dcb7deed617c19ac1de4a68. after waiting 0 ms 2024-11-08T05:13:25,717 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,fff,1731042805591.65d5312dbd2f96bf3b16bdfd7504dff6. after waiting 0 ms 2024-11-08T05:13:25,717 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,ggg,1731042805591.9b3274d80dcb7deed617c19ac1de4a68. 2024-11-08T05:13:25,717 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,fff,1731042805591.65d5312dbd2f96bf3b16bdfd7504dff6. 2024-11-08T05:13:25,717 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,ggg,1731042805591.9b3274d80dcb7deed617c19ac1de4a68. 2024-11-08T05:13:25,717 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,fff,1731042805591.65d5312dbd2f96bf3b16bdfd7504dff6. 2024-11-08T05:13:25,717 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 65d5312dbd2f96bf3b16bdfd7504dff6: Waiting for close lock at 1731042805717Disabling compacts and flushes for region at 1731042805717Disabling writes for close at 1731042805717Writing region close event to WAL at 1731042805717Closed at 1731042805717 2024-11-08T05:13:25,717 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 9b3274d80dcb7deed617c19ac1de4a68: Waiting for close lock at 1731042805717Disabling compacts and flushes for region at 1731042805717Disabling writes for close at 1731042805717Writing region close event to WAL at 1731042805717Closed at 1731042805717 2024-11-08T05:13:25,718 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => fe1545b16a81758dcb9430dbf98c4ab9, NAME => 'test,iii,1731042805591.fe1545b16a81758dcb9430dbf98c4ab9.', STARTKEY => 'iii', ENDKEY => 'jjj'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c 2024-11-08T05:13:25,719 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 2d23e3e11ecc480e314e989d1bb8d0c3, NAME => 'test,hhh,1731042805591.2d23e3e11ecc480e314e989d1bb8d0c3.', STARTKEY => 'hhh', ENDKEY => 'iii'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c 2024-11-08T05:13:25,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37587 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-08T05:13:25,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741846_1022 (size=45) 2024-11-08T05:13:25,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741845_1021 (size=45) 2024-11-08T05:13:25,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741846_1022 (size=45) 2024-11-08T05:13:25,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741845_1021 (size=45) 2024-11-08T05:13:25,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34565 is added to blk_1073741846_1022 (size=45) 2024-11-08T05:13:25,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34565 is added to blk_1073741845_1021 (size=45) 2024-11-08T05:13:25,749 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,hhh,1731042805591.2d23e3e11ecc480e314e989d1bb8d0c3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T05:13:25,749 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 2d23e3e11ecc480e314e989d1bb8d0c3, disabling compactions & flushes 2024-11-08T05:13:25,749 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,hhh,1731042805591.2d23e3e11ecc480e314e989d1bb8d0c3. 2024-11-08T05:13:25,750 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,hhh,1731042805591.2d23e3e11ecc480e314e989d1bb8d0c3. 2024-11-08T05:13:25,750 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,hhh,1731042805591.2d23e3e11ecc480e314e989d1bb8d0c3. after waiting 0 ms 2024-11-08T05:13:25,750 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,hhh,1731042805591.2d23e3e11ecc480e314e989d1bb8d0c3. 2024-11-08T05:13:25,750 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,hhh,1731042805591.2d23e3e11ecc480e314e989d1bb8d0c3. 2024-11-08T05:13:25,750 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 2d23e3e11ecc480e314e989d1bb8d0c3: Waiting for close lock at 1731042805749Disabling compacts and flushes for region at 1731042805749Disabling writes for close at 1731042805750 (+1 ms)Writing region close event to WAL at 1731042805750Closed at 1731042805750 2024-11-08T05:13:25,750 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => a12ea80f78b339cfbfc0e18b5338df0c, NAME => 'test,jjj,1731042805591.a12ea80f78b339cfbfc0e18b5338df0c.', STARTKEY => 'jjj', ENDKEY => 'kkk'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c 2024-11-08T05:13:25,750 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,iii,1731042805591.fe1545b16a81758dcb9430dbf98c4ab9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T05:13:25,751 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing fe1545b16a81758dcb9430dbf98c4ab9, disabling compactions & flushes 2024-11-08T05:13:25,751 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,iii,1731042805591.fe1545b16a81758dcb9430dbf98c4ab9. 2024-11-08T05:13:25,751 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,iii,1731042805591.fe1545b16a81758dcb9430dbf98c4ab9. 2024-11-08T05:13:25,751 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,iii,1731042805591.fe1545b16a81758dcb9430dbf98c4ab9. after waiting 0 ms 2024-11-08T05:13:25,751 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,iii,1731042805591.fe1545b16a81758dcb9430dbf98c4ab9. 2024-11-08T05:13:25,751 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,iii,1731042805591.fe1545b16a81758dcb9430dbf98c4ab9. 2024-11-08T05:13:25,751 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for fe1545b16a81758dcb9430dbf98c4ab9: Waiting for close lock at 1731042805751Disabling compacts and flushes for region at 1731042805751Disabling writes for close at 1731042805751Writing region close event to WAL at 1731042805751Closed at 1731042805751 2024-11-08T05:13:25,752 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 0e1c63297d3a8aea2fdbd1842301c811, NAME => 'test,kkk,1731042805591.0e1c63297d3a8aea2fdbd1842301c811.', STARTKEY => 'kkk', ENDKEY => 'lll'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c 2024-11-08T05:13:25,755 WARN [IPC Server handler 2 on default port 45429 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-08T05:13:25,755 WARN [IPC Server handler 4 on default port 45429 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-08T05:13:25,755 WARN [IPC Server handler 2 on default port 45429 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-08T05:13:25,755 WARN [IPC Server handler 4 on default port 45429 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-08T05:13:25,755 WARN [IPC Server handler 4 on default port 45429 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-08T05:13:25,755 WARN [IPC Server handler 2 on default port 45429 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-08T05:13:25,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741847_1023 (size=45) 2024-11-08T05:13:25,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741848_1024 (size=45) 2024-11-08T05:13:25,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741847_1023 (size=45) 2024-11-08T05:13:25,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741848_1024 (size=45) 2024-11-08T05:13:25,764 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,jjj,1731042805591.a12ea80f78b339cfbfc0e18b5338df0c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T05:13:25,764 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing a12ea80f78b339cfbfc0e18b5338df0c, disabling compactions & flushes 2024-11-08T05:13:25,764 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,jjj,1731042805591.a12ea80f78b339cfbfc0e18b5338df0c. 2024-11-08T05:13:25,764 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,jjj,1731042805591.a12ea80f78b339cfbfc0e18b5338df0c. 2024-11-08T05:13:25,764 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,jjj,1731042805591.a12ea80f78b339cfbfc0e18b5338df0c. after waiting 0 ms 2024-11-08T05:13:25,764 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,jjj,1731042805591.a12ea80f78b339cfbfc0e18b5338df0c. 2024-11-08T05:13:25,764 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,jjj,1731042805591.a12ea80f78b339cfbfc0e18b5338df0c. 2024-11-08T05:13:25,764 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for a12ea80f78b339cfbfc0e18b5338df0c: Waiting for close lock at 1731042805764Disabling compacts and flushes for region at 1731042805764Disabling writes for close at 1731042805764Writing region close event to WAL at 1731042805764Closed at 1731042805764 2024-11-08T05:13:25,765 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 40440b5f39bb5421276aa0b156481a9e, NAME => 'test,lll,1731042805591.40440b5f39bb5421276aa0b156481a9e.', STARTKEY => 'lll', ENDKEY => 'mmm'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c 2024-11-08T05:13:25,769 WARN [IPC Server handler 2 on default port 45429 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-08T05:13:25,770 WARN [IPC Server handler 2 on default port 45429 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-08T05:13:25,770 WARN [IPC Server handler 2 on default port 45429 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-08T05:13:25,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741849_1025 (size=45) 2024-11-08T05:13:25,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741849_1025 (size=45) 2024-11-08T05:13:25,779 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,lll,1731042805591.40440b5f39bb5421276aa0b156481a9e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T05:13:25,779 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 40440b5f39bb5421276aa0b156481a9e, disabling compactions & flushes 2024-11-08T05:13:25,779 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,lll,1731042805591.40440b5f39bb5421276aa0b156481a9e. 2024-11-08T05:13:25,780 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,lll,1731042805591.40440b5f39bb5421276aa0b156481a9e. 2024-11-08T05:13:25,780 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,lll,1731042805591.40440b5f39bb5421276aa0b156481a9e. after waiting 0 ms 2024-11-08T05:13:25,780 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,lll,1731042805591.40440b5f39bb5421276aa0b156481a9e. 2024-11-08T05:13:25,780 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,lll,1731042805591.40440b5f39bb5421276aa0b156481a9e. 2024-11-08T05:13:25,780 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 40440b5f39bb5421276aa0b156481a9e: Waiting for close lock at 1731042805779Disabling compacts and flushes for region at 1731042805779Disabling writes for close at 1731042805780 (+1 ms)Writing region close event to WAL at 1731042805780Closed at 1731042805780 2024-11-08T05:13:25,780 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 27c9aaf34aaf98cd5bc8030630ae7fe3, NAME => 'test,mmm,1731042805591.27c9aaf34aaf98cd5bc8030630ae7fe3.', STARTKEY => 'mmm', ENDKEY => 'nnn'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c 2024-11-08T05:13:25,785 WARN [IPC Server handler 0 on default port 45429 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-08T05:13:25,785 WARN [IPC Server handler 0 on default port 45429 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-08T05:13:25,785 WARN [IPC Server handler 0 on default port 45429 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-08T05:13:25,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741850_1026 (size=45) 2024-11-08T05:13:25,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741850_1026 (size=45) 2024-11-08T05:13:25,794 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,mmm,1731042805591.27c9aaf34aaf98cd5bc8030630ae7fe3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T05:13:25,794 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 27c9aaf34aaf98cd5bc8030630ae7fe3, disabling compactions & flushes 2024-11-08T05:13:25,794 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,mmm,1731042805591.27c9aaf34aaf98cd5bc8030630ae7fe3. 2024-11-08T05:13:25,794 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,mmm,1731042805591.27c9aaf34aaf98cd5bc8030630ae7fe3. 2024-11-08T05:13:25,794 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,mmm,1731042805591.27c9aaf34aaf98cd5bc8030630ae7fe3. after waiting 0 ms 2024-11-08T05:13:25,794 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,mmm,1731042805591.27c9aaf34aaf98cd5bc8030630ae7fe3. 2024-11-08T05:13:25,794 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,mmm,1731042805591.27c9aaf34aaf98cd5bc8030630ae7fe3. 2024-11-08T05:13:25,795 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 27c9aaf34aaf98cd5bc8030630ae7fe3: Waiting for close lock at 1731042805794Disabling compacts and flushes for region at 1731042805794Disabling writes for close at 1731042805794Writing region close event to WAL at 1731042805794Closed at 1731042805794 2024-11-08T05:13:25,796 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 868527ec322eba1d7cbd6371701f7ad2, NAME => 'test,nnn,1731042805591.868527ec322eba1d7cbd6371701f7ad2.', STARTKEY => 'nnn', ENDKEY => 'ooo'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c 2024-11-08T05:13:25,800 WARN [IPC Server handler 4 on default port 45429 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-08T05:13:25,800 WARN [IPC Server handler 4 on default port 45429 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-08T05:13:25,800 WARN [IPC Server handler 4 on default port 45429 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-08T05:13:25,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741851_1027 (size=45) 2024-11-08T05:13:25,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741851_1027 (size=45) 2024-11-08T05:13:25,810 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,nnn,1731042805591.868527ec322eba1d7cbd6371701f7ad2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T05:13:25,810 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 868527ec322eba1d7cbd6371701f7ad2, disabling compactions & flushes 2024-11-08T05:13:25,810 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,nnn,1731042805591.868527ec322eba1d7cbd6371701f7ad2. 2024-11-08T05:13:25,810 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,nnn,1731042805591.868527ec322eba1d7cbd6371701f7ad2. 2024-11-08T05:13:25,810 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,nnn,1731042805591.868527ec322eba1d7cbd6371701f7ad2. after waiting 0 ms 2024-11-08T05:13:25,810 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,nnn,1731042805591.868527ec322eba1d7cbd6371701f7ad2. 2024-11-08T05:13:25,810 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,nnn,1731042805591.868527ec322eba1d7cbd6371701f7ad2. 2024-11-08T05:13:25,810 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 868527ec322eba1d7cbd6371701f7ad2: Waiting for close lock at 1731042805810Disabling compacts and flushes for region at 1731042805810Disabling writes for close at 1731042805810Writing region close event to WAL at 1731042805810Closed at 1731042805810 2024-11-08T05:13:25,811 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => d951febea78018bb9f10f07d8788575d, NAME => 'test,ooo,1731042805591.d951febea78018bb9f10f07d8788575d.', STARTKEY => 'ooo', ENDKEY => 'ppp'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c 2024-11-08T05:13:25,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741852_1028 (size=45) 2024-11-08T05:13:25,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741852_1028 (size=45) 2024-11-08T05:13:25,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34565 is added to blk_1073741852_1028 (size=45) 2024-11-08T05:13:25,823 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,ooo,1731042805591.d951febea78018bb9f10f07d8788575d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T05:13:25,823 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing d951febea78018bb9f10f07d8788575d, disabling compactions & flushes 2024-11-08T05:13:25,823 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,ooo,1731042805591.d951febea78018bb9f10f07d8788575d. 2024-11-08T05:13:25,823 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ooo,1731042805591.d951febea78018bb9f10f07d8788575d. 2024-11-08T05:13:25,823 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,ooo,1731042805591.d951febea78018bb9f10f07d8788575d. after waiting 0 ms 2024-11-08T05:13:25,823 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,ooo,1731042805591.d951febea78018bb9f10f07d8788575d. 2024-11-08T05:13:25,824 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,ooo,1731042805591.d951febea78018bb9f10f07d8788575d. 2024-11-08T05:13:25,824 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for d951febea78018bb9f10f07d8788575d: Waiting for close lock at 1731042805823Disabling compacts and flushes for region at 1731042805823Disabling writes for close at 1731042805823Writing region close event to WAL at 1731042805823Closed at 1731042805823 2024-11-08T05:13:25,825 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => b0ebd9a9794acc56e430b36d1f17621c, NAME => 'test,ppp,1731042805591.b0ebd9a9794acc56e430b36d1f17621c.', STARTKEY => 'ppp', ENDKEY => 'qqq'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c 2024-11-08T05:13:25,828 WARN [IPC Server handler 4 on default port 45429 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-08T05:13:25,828 WARN [IPC Server handler 4 on default port 45429 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-08T05:13:25,828 WARN [IPC Server handler 4 on default port 45429 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-08T05:13:25,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741853_1029 (size=45) 2024-11-08T05:13:25,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741853_1029 (size=45) 2024-11-08T05:13:25,835 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,ppp,1731042805591.b0ebd9a9794acc56e430b36d1f17621c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T05:13:25,835 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing b0ebd9a9794acc56e430b36d1f17621c, disabling compactions & flushes 2024-11-08T05:13:25,835 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,ppp,1731042805591.b0ebd9a9794acc56e430b36d1f17621c. 2024-11-08T05:13:25,835 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ppp,1731042805591.b0ebd9a9794acc56e430b36d1f17621c. 2024-11-08T05:13:25,835 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,ppp,1731042805591.b0ebd9a9794acc56e430b36d1f17621c. after waiting 0 ms 2024-11-08T05:13:25,835 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,ppp,1731042805591.b0ebd9a9794acc56e430b36d1f17621c. 2024-11-08T05:13:25,835 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,ppp,1731042805591.b0ebd9a9794acc56e430b36d1f17621c. 2024-11-08T05:13:25,835 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for b0ebd9a9794acc56e430b36d1f17621c: Waiting for close lock at 1731042805835Disabling compacts and flushes for region at 1731042805835Disabling writes for close at 1731042805835Writing region close event to WAL at 1731042805835Closed at 1731042805835 2024-11-08T05:13:25,836 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 7830dc6b1acea6e1d468f8008d8a82b9, NAME => 'test,qqq,1731042805591.7830dc6b1acea6e1d468f8008d8a82b9.', STARTKEY => 'qqq', ENDKEY => 'rrr'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c 2024-11-08T05:13:25,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741854_1030 (size=45) 2024-11-08T05:13:25,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741854_1030 (size=45) 2024-11-08T05:13:25,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34565 is added to blk_1073741854_1030 (size=45) 2024-11-08T05:13:25,848 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,qqq,1731042805591.7830dc6b1acea6e1d468f8008d8a82b9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T05:13:25,848 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 7830dc6b1acea6e1d468f8008d8a82b9, disabling compactions & flushes 2024-11-08T05:13:25,848 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,qqq,1731042805591.7830dc6b1acea6e1d468f8008d8a82b9. 2024-11-08T05:13:25,848 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,qqq,1731042805591.7830dc6b1acea6e1d468f8008d8a82b9. 2024-11-08T05:13:25,848 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,qqq,1731042805591.7830dc6b1acea6e1d468f8008d8a82b9. after waiting 0 ms 2024-11-08T05:13:25,848 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,qqq,1731042805591.7830dc6b1acea6e1d468f8008d8a82b9. 2024-11-08T05:13:25,848 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,qqq,1731042805591.7830dc6b1acea6e1d468f8008d8a82b9. 2024-11-08T05:13:25,848 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 7830dc6b1acea6e1d468f8008d8a82b9: Waiting for close lock at 1731042805848Disabling compacts and flushes for region at 1731042805848Disabling writes for close at 1731042805848Writing region close event to WAL at 1731042805848Closed at 1731042805848 2024-11-08T05:13:25,849 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 12f1148a7b8db3e4f515337b8b7746fa, NAME => 'test,rrr,1731042805591.12f1148a7b8db3e4f515337b8b7746fa.', STARTKEY => 'rrr', ENDKEY => 'sss'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c 2024-11-08T05:13:25,852 WARN [IPC Server handler 4 on default port 45429 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-08T05:13:25,853 WARN [IPC Server handler 4 on default port 45429 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-08T05:13:25,853 WARN [IPC Server handler 4 on default port 45429 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-08T05:13:25,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741855_1031 (size=45) 2024-11-08T05:13:25,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741855_1031 (size=45) 2024-11-08T05:13:25,858 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,rrr,1731042805591.12f1148a7b8db3e4f515337b8b7746fa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T05:13:25,859 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 12f1148a7b8db3e4f515337b8b7746fa, disabling compactions & flushes 2024-11-08T05:13:25,859 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,rrr,1731042805591.12f1148a7b8db3e4f515337b8b7746fa. 2024-11-08T05:13:25,859 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,rrr,1731042805591.12f1148a7b8db3e4f515337b8b7746fa. 2024-11-08T05:13:25,859 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,rrr,1731042805591.12f1148a7b8db3e4f515337b8b7746fa. after waiting 0 ms 2024-11-08T05:13:25,859 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,rrr,1731042805591.12f1148a7b8db3e4f515337b8b7746fa. 2024-11-08T05:13:25,859 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,rrr,1731042805591.12f1148a7b8db3e4f515337b8b7746fa. 2024-11-08T05:13:25,859 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 12f1148a7b8db3e4f515337b8b7746fa: Waiting for close lock at 1731042805859Disabling compacts and flushes for region at 1731042805859Disabling writes for close at 1731042805859Writing region close event to WAL at 1731042805859Closed at 1731042805859 2024-11-08T05:13:25,859 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => e26dcb2339b364f3699d110c15cd4bb3, NAME => 'test,sss,1731042805591.e26dcb2339b364f3699d110c15cd4bb3.', STARTKEY => 'sss', ENDKEY => 'ttt'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c 2024-11-08T05:13:25,863 WARN [IPC Server handler 1 on default port 45429 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-08T05:13:25,863 WARN [IPC Server handler 1 on default port 45429 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-08T05:13:25,863 WARN [IPC Server handler 1 on default port 45429 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-08T05:13:25,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741856_1032 (size=45) 2024-11-08T05:13:25,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741856_1032 (size=45) 2024-11-08T05:13:25,871 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,sss,1731042805591.e26dcb2339b364f3699d110c15cd4bb3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T05:13:25,871 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing e26dcb2339b364f3699d110c15cd4bb3, disabling compactions & flushes 2024-11-08T05:13:25,871 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,sss,1731042805591.e26dcb2339b364f3699d110c15cd4bb3. 2024-11-08T05:13:25,871 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,sss,1731042805591.e26dcb2339b364f3699d110c15cd4bb3. 2024-11-08T05:13:25,871 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,sss,1731042805591.e26dcb2339b364f3699d110c15cd4bb3. after waiting 0 ms 2024-11-08T05:13:25,871 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,sss,1731042805591.e26dcb2339b364f3699d110c15cd4bb3. 2024-11-08T05:13:25,871 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,sss,1731042805591.e26dcb2339b364f3699d110c15cd4bb3. 2024-11-08T05:13:25,871 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for e26dcb2339b364f3699d110c15cd4bb3: Waiting for close lock at 1731042805871Disabling compacts and flushes for region at 1731042805871Disabling writes for close at 1731042805871Writing region close event to WAL at 1731042805871Closed at 1731042805871 2024-11-08T05:13:25,872 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 445b14856951586918c91d4ba342f1ff, NAME => 'test,ttt,1731042805591.445b14856951586918c91d4ba342f1ff.', STARTKEY => 'ttt', ENDKEY => 'uuu'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c 2024-11-08T05:13:25,876 WARN [IPC Server handler 3 on default port 45429 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-08T05:13:25,877 WARN [IPC Server handler 3 on default port 45429 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-08T05:13:25,877 WARN [IPC Server handler 3 on default port 45429 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-08T05:13:25,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741857_1033 (size=45) 2024-11-08T05:13:25,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741857_1033 (size=45) 2024-11-08T05:13:25,883 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,ttt,1731042805591.445b14856951586918c91d4ba342f1ff.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T05:13:25,884 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 445b14856951586918c91d4ba342f1ff, disabling compactions & flushes 2024-11-08T05:13:25,884 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,ttt,1731042805591.445b14856951586918c91d4ba342f1ff. 2024-11-08T05:13:25,884 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ttt,1731042805591.445b14856951586918c91d4ba342f1ff. 2024-11-08T05:13:25,884 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,ttt,1731042805591.445b14856951586918c91d4ba342f1ff. after waiting 0 ms 2024-11-08T05:13:25,884 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,ttt,1731042805591.445b14856951586918c91d4ba342f1ff. 2024-11-08T05:13:25,884 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,ttt,1731042805591.445b14856951586918c91d4ba342f1ff. 2024-11-08T05:13:25,884 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 445b14856951586918c91d4ba342f1ff: Waiting for close lock at 1731042805884Disabling compacts and flushes for region at 1731042805884Disabling writes for close at 1731042805884Writing region close event to WAL at 1731042805884Closed at 1731042805884 2024-11-08T05:13:25,885 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 14efcc1b9822cc74f4d4a96648555167, NAME => 'test,uuu,1731042805591.14efcc1b9822cc74f4d4a96648555167.', STARTKEY => 'uuu', ENDKEY => 'vvv'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c 2024-11-08T05:13:25,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741858_1034 (size=45) 2024-11-08T05:13:25,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741858_1034 (size=45) 2024-11-08T05:13:25,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34565 is added to blk_1073741858_1034 (size=45) 2024-11-08T05:13:25,895 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,uuu,1731042805591.14efcc1b9822cc74f4d4a96648555167.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T05:13:25,895 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 14efcc1b9822cc74f4d4a96648555167, disabling compactions & flushes 2024-11-08T05:13:25,895 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,uuu,1731042805591.14efcc1b9822cc74f4d4a96648555167. 2024-11-08T05:13:25,895 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,uuu,1731042805591.14efcc1b9822cc74f4d4a96648555167. 2024-11-08T05:13:25,895 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,uuu,1731042805591.14efcc1b9822cc74f4d4a96648555167. after waiting 0 ms 2024-11-08T05:13:25,895 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,uuu,1731042805591.14efcc1b9822cc74f4d4a96648555167. 2024-11-08T05:13:25,895 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,uuu,1731042805591.14efcc1b9822cc74f4d4a96648555167. 2024-11-08T05:13:25,895 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 14efcc1b9822cc74f4d4a96648555167: Waiting for close lock at 1731042805895Disabling compacts and flushes for region at 1731042805895Disabling writes for close at 1731042805895Writing region close event to WAL at 1731042805895Closed at 1731042805895 2024-11-08T05:13:25,896 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 7c8f72849b2e1719ed903afc73e0571a, NAME => 'test,vvv,1731042805591.7c8f72849b2e1719ed903afc73e0571a.', STARTKEY => 'vvv', ENDKEY => 'www'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c 2024-11-08T05:13:25,900 WARN [IPC Server handler 3 on default port 45429 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-08T05:13:25,900 WARN [IPC Server handler 3 on default port 45429 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-08T05:13:25,900 WARN [IPC Server handler 3 on default port 45429 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-08T05:13:25,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741859_1035 (size=45) 2024-11-08T05:13:25,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741859_1035 (size=45) 2024-11-08T05:13:25,912 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,vvv,1731042805591.7c8f72849b2e1719ed903afc73e0571a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T05:13:25,912 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 7c8f72849b2e1719ed903afc73e0571a, disabling compactions & flushes 2024-11-08T05:13:25,912 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,vvv,1731042805591.7c8f72849b2e1719ed903afc73e0571a. 2024-11-08T05:13:25,912 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,vvv,1731042805591.7c8f72849b2e1719ed903afc73e0571a. 2024-11-08T05:13:25,912 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,vvv,1731042805591.7c8f72849b2e1719ed903afc73e0571a. after waiting 0 ms 2024-11-08T05:13:25,912 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,vvv,1731042805591.7c8f72849b2e1719ed903afc73e0571a. 2024-11-08T05:13:25,912 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,vvv,1731042805591.7c8f72849b2e1719ed903afc73e0571a. 2024-11-08T05:13:25,913 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 7c8f72849b2e1719ed903afc73e0571a: Waiting for close lock at 1731042805912Disabling compacts and flushes for region at 1731042805912Disabling writes for close at 1731042805912Writing region close event to WAL at 1731042805912Closed at 1731042805912 2024-11-08T05:13:25,913 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 81806cb184c70c9dc0df408694092d85, NAME => 'test,www,1731042805591.81806cb184c70c9dc0df408694092d85.', STARTKEY => 'www', ENDKEY => 'xxx'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c 2024-11-08T05:13:25,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741860_1036 (size=45) 2024-11-08T05:13:25,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741860_1036 (size=45) 2024-11-08T05:13:25,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34565 is added to blk_1073741860_1036 (size=45) 2024-11-08T05:13:25,926 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,www,1731042805591.81806cb184c70c9dc0df408694092d85.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T05:13:25,926 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 81806cb184c70c9dc0df408694092d85, disabling compactions & flushes 2024-11-08T05:13:25,926 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,www,1731042805591.81806cb184c70c9dc0df408694092d85. 2024-11-08T05:13:25,926 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,www,1731042805591.81806cb184c70c9dc0df408694092d85. 2024-11-08T05:13:25,926 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,www,1731042805591.81806cb184c70c9dc0df408694092d85. after waiting 0 ms 2024-11-08T05:13:25,927 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,www,1731042805591.81806cb184c70c9dc0df408694092d85. 2024-11-08T05:13:25,927 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,www,1731042805591.81806cb184c70c9dc0df408694092d85. 2024-11-08T05:13:25,927 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 81806cb184c70c9dc0df408694092d85: Waiting for close lock at 1731042805926Disabling compacts and flushes for region at 1731042805926Disabling writes for close at 1731042805926Writing region close event to WAL at 1731042805927 (+1 ms)Closed at 1731042805927 2024-11-08T05:13:25,927 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 87b3455cffd1f4e06e5901b3bcbed9dc, NAME => 'test,xxx,1731042805591.87b3455cffd1f4e06e5901b3bcbed9dc.', STARTKEY => 'xxx', ENDKEY => 'yyy'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c 2024-11-08T05:13:25,931 WARN [IPC Server handler 3 on default port 45429 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-08T05:13:25,931 WARN [IPC Server handler 3 on default port 45429 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-08T05:13:25,932 WARN [IPC Server handler 3 on default port 45429 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-08T05:13:25,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741861_1037 (size=45) 2024-11-08T05:13:25,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741861_1037 (size=45) 2024-11-08T05:13:25,938 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,xxx,1731042805591.87b3455cffd1f4e06e5901b3bcbed9dc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T05:13:25,938 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 87b3455cffd1f4e06e5901b3bcbed9dc, disabling compactions & flushes 2024-11-08T05:13:25,938 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,xxx,1731042805591.87b3455cffd1f4e06e5901b3bcbed9dc. 2024-11-08T05:13:25,938 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,xxx,1731042805591.87b3455cffd1f4e06e5901b3bcbed9dc. 2024-11-08T05:13:25,938 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,xxx,1731042805591.87b3455cffd1f4e06e5901b3bcbed9dc. after waiting 0 ms 2024-11-08T05:13:25,938 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,xxx,1731042805591.87b3455cffd1f4e06e5901b3bcbed9dc. 2024-11-08T05:13:25,938 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,xxx,1731042805591.87b3455cffd1f4e06e5901b3bcbed9dc. 2024-11-08T05:13:25,938 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 87b3455cffd1f4e06e5901b3bcbed9dc: Waiting for close lock at 1731042805938Disabling compacts and flushes for region at 1731042805938Disabling writes for close at 1731042805938Writing region close event to WAL at 1731042805938Closed at 1731042805938 2024-11-08T05:13:25,939 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => de358f808110a48fa32e0533fefdc95c, NAME => 'test,yyy,1731042805591.de358f808110a48fa32e0533fefdc95c.', STARTKEY => 'yyy', ENDKEY => 'zzz'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c 2024-11-08T05:13:25,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37587 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-08T05:13:25,944 WARN [IPC Server handler 2 on default port 45429 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-08T05:13:25,944 WARN [IPC Server handler 2 on default port 45429 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-08T05:13:25,944 WARN [IPC Server handler 2 on default port 45429 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-08T05:13:25,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741862_1038 (size=45) 2024-11-08T05:13:25,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741862_1038 (size=45) 2024-11-08T05:13:25,950 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,yyy,1731042805591.de358f808110a48fa32e0533fefdc95c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T05:13:25,950 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing de358f808110a48fa32e0533fefdc95c, disabling compactions & flushes 2024-11-08T05:13:25,950 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,yyy,1731042805591.de358f808110a48fa32e0533fefdc95c. 2024-11-08T05:13:25,950 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,yyy,1731042805591.de358f808110a48fa32e0533fefdc95c. 2024-11-08T05:13:25,950 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,yyy,1731042805591.de358f808110a48fa32e0533fefdc95c. after waiting 0 ms 2024-11-08T05:13:25,950 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,yyy,1731042805591.de358f808110a48fa32e0533fefdc95c. 2024-11-08T05:13:25,950 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,yyy,1731042805591.de358f808110a48fa32e0533fefdc95c. 2024-11-08T05:13:25,950 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for de358f808110a48fa32e0533fefdc95c: Waiting for close lock at 1731042805950Disabling compacts and flushes for region at 1731042805950Disabling writes for close at 1731042805950Writing region close event to WAL at 1731042805950Closed at 1731042805950 2024-11-08T05:13:25,951 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => e61dc9e059921fddc5ad7f6f7971acc6, NAME => 'test,zzz,1731042805591.e61dc9e059921fddc5ad7f6f7971acc6.', STARTKEY => 'zzz', ENDKEY => ''}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c 2024-11-08T05:13:25,955 WARN [IPC Server handler 0 on default port 45429 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-08T05:13:25,955 WARN [IPC Server handler 0 on default port 45429 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-08T05:13:25,955 WARN [IPC Server handler 0 on default port 45429 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-08T05:13:25,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741863_1039 (size=42) 2024-11-08T05:13:25,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741863_1039 (size=42) 2024-11-08T05:13:25,962 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,zzz,1731042805591.e61dc9e059921fddc5ad7f6f7971acc6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T05:13:25,963 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing e61dc9e059921fddc5ad7f6f7971acc6, disabling compactions & flushes 2024-11-08T05:13:25,963 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,zzz,1731042805591.e61dc9e059921fddc5ad7f6f7971acc6. 2024-11-08T05:13:25,963 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,zzz,1731042805591.e61dc9e059921fddc5ad7f6f7971acc6. 2024-11-08T05:13:25,963 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,zzz,1731042805591.e61dc9e059921fddc5ad7f6f7971acc6. after waiting 0 ms 2024-11-08T05:13:25,963 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,zzz,1731042805591.e61dc9e059921fddc5ad7f6f7971acc6. 2024-11-08T05:13:25,963 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,zzz,1731042805591.e61dc9e059921fddc5ad7f6f7971acc6. 2024-11-08T05:13:25,963 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for e61dc9e059921fddc5ad7f6f7971acc6: Waiting for close lock at 1731042805963Disabling compacts and flushes for region at 1731042805963Disabling writes for close at 1731042805963Writing region close event to WAL at 1731042805963Closed at 1731042805963 2024-11-08T05:13:26,066 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,,1731042805591.6240e38715889f3bb77bb89478c81a6c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T05:13:26,066 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 6240e38715889f3bb77bb89478c81a6c, disabling compactions & flushes 2024-11-08T05:13:26,066 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,,1731042805591.6240e38715889f3bb77bb89478c81a6c. 2024-11-08T05:13:26,066 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,,1731042805591.6240e38715889f3bb77bb89478c81a6c. 2024-11-08T05:13:26,066 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,,1731042805591.6240e38715889f3bb77bb89478c81a6c. after waiting 0 ms 2024-11-08T05:13:26,066 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,,1731042805591.6240e38715889f3bb77bb89478c81a6c. 2024-11-08T05:13:26,066 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,,1731042805591.6240e38715889f3bb77bb89478c81a6c. 2024-11-08T05:13:26,067 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 6240e38715889f3bb77bb89478c81a6c: Waiting for close lock at 1731042806066Disabling compacts and flushes for region at 1731042806066Disabling writes for close at 1731042806066Writing region close event to WAL at 1731042806066Closed at 1731042806066 2024-11-08T05:13:26,164 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,kkk,1731042805591.0e1c63297d3a8aea2fdbd1842301c811.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T05:13:26,164 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 0e1c63297d3a8aea2fdbd1842301c811, disabling compactions & flushes 2024-11-08T05:13:26,164 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,kkk,1731042805591.0e1c63297d3a8aea2fdbd1842301c811. 2024-11-08T05:13:26,164 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,kkk,1731042805591.0e1c63297d3a8aea2fdbd1842301c811. 2024-11-08T05:13:26,164 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,kkk,1731042805591.0e1c63297d3a8aea2fdbd1842301c811. after waiting 0 ms 2024-11-08T05:13:26,164 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,kkk,1731042805591.0e1c63297d3a8aea2fdbd1842301c811. 2024-11-08T05:13:26,164 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,kkk,1731042805591.0e1c63297d3a8aea2fdbd1842301c811. 2024-11-08T05:13:26,164 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 0e1c63297d3a8aea2fdbd1842301c811: Waiting for close lock at 1731042806164Disabling compacts and flushes for region at 1731042806164Disabling writes for close at 1731042806164Writing region close event to WAL at 1731042806164Closed at 1731042806164 2024-11-08T05:13:26,167 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T05:13:26,173 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ccc,1731042805591.68bbd0353f97ee12228f7798b0b987ea.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731042806167"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731042806167"}]},"ts":"1731042806167"} 2024-11-08T05:13:26,173 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,bbb,1731042805591.eeaad0e7e7aa66f0b51c273c0aaf953c.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731042806167"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731042806167"}]},"ts":"1731042806167"} 2024-11-08T05:13:26,174 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ddd,1731042805591.64ddf686fed30cdb23b2e874efec598d.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731042806167"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731042806167"}]},"ts":"1731042806167"} 2024-11-08T05:13:26,174 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,eee,1731042805591.d7680a9929f15640ca61551f186f33c1.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731042806167"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731042806167"}]},"ts":"1731042806167"} 2024-11-08T05:13:26,174 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,fff,1731042805591.65d5312dbd2f96bf3b16bdfd7504dff6.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731042806167"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731042806167"}]},"ts":"1731042806167"} 2024-11-08T05:13:26,174 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ggg,1731042805591.9b3274d80dcb7deed617c19ac1de4a68.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731042806167"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731042806167"}]},"ts":"1731042806167"} 2024-11-08T05:13:26,174 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,hhh,1731042805591.2d23e3e11ecc480e314e989d1bb8d0c3.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731042806167"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731042806167"}]},"ts":"1731042806167"} 2024-11-08T05:13:26,174 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,iii,1731042805591.fe1545b16a81758dcb9430dbf98c4ab9.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731042806167"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731042806167"}]},"ts":"1731042806167"} 2024-11-08T05:13:26,175 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,jjj,1731042805591.a12ea80f78b339cfbfc0e18b5338df0c.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731042806167"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731042806167"}]},"ts":"1731042806167"} 2024-11-08T05:13:26,175 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,lll,1731042805591.40440b5f39bb5421276aa0b156481a9e.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731042806167"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731042806167"}]},"ts":"1731042806167"} 2024-11-08T05:13:26,175 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,mmm,1731042805591.27c9aaf34aaf98cd5bc8030630ae7fe3.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731042806167"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731042806167"}]},"ts":"1731042806167"} 2024-11-08T05:13:26,175 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,nnn,1731042805591.868527ec322eba1d7cbd6371701f7ad2.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731042806167"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731042806167"}]},"ts":"1731042806167"} 2024-11-08T05:13:26,175 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ooo,1731042805591.d951febea78018bb9f10f07d8788575d.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731042806167"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731042806167"}]},"ts":"1731042806167"} 2024-11-08T05:13:26,175 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ppp,1731042805591.b0ebd9a9794acc56e430b36d1f17621c.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731042806167"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731042806167"}]},"ts":"1731042806167"} 2024-11-08T05:13:26,175 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,qqq,1731042805591.7830dc6b1acea6e1d468f8008d8a82b9.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731042806167"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731042806167"}]},"ts":"1731042806167"} 2024-11-08T05:13:26,176 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,rrr,1731042805591.12f1148a7b8db3e4f515337b8b7746fa.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731042806167"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731042806167"}]},"ts":"1731042806167"} 2024-11-08T05:13:26,176 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,sss,1731042805591.e26dcb2339b364f3699d110c15cd4bb3.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731042806167"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731042806167"}]},"ts":"1731042806167"} 2024-11-08T05:13:26,176 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ttt,1731042805591.445b14856951586918c91d4ba342f1ff.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731042806167"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731042806167"}]},"ts":"1731042806167"} 2024-11-08T05:13:26,176 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,uuu,1731042805591.14efcc1b9822cc74f4d4a96648555167.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731042806167"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731042806167"}]},"ts":"1731042806167"} 2024-11-08T05:13:26,176 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,vvv,1731042805591.7c8f72849b2e1719ed903afc73e0571a.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731042806167"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731042806167"}]},"ts":"1731042806167"} 2024-11-08T05:13:26,176 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,www,1731042805591.81806cb184c70c9dc0df408694092d85.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731042806167"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731042806167"}]},"ts":"1731042806167"} 2024-11-08T05:13:26,176 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,xxx,1731042805591.87b3455cffd1f4e06e5901b3bcbed9dc.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731042806167"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731042806167"}]},"ts":"1731042806167"} 2024-11-08T05:13:26,177 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,yyy,1731042805591.de358f808110a48fa32e0533fefdc95c.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731042806167"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731042806167"}]},"ts":"1731042806167"} 2024-11-08T05:13:26,177 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,zzz,1731042805591.e61dc9e059921fddc5ad7f6f7971acc6.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1731042806167"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731042806167"}]},"ts":"1731042806167"} 2024-11-08T05:13:26,177 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,,1731042805591.6240e38715889f3bb77bb89478c81a6c.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1731042806167"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731042806167"}]},"ts":"1731042806167"} 2024-11-08T05:13:26,177 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,kkk,1731042805591.0e1c63297d3a8aea2fdbd1842301c811.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731042806167"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731042806167"}]},"ts":"1731042806167"} 2024-11-08T05:13:26,215 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 26 regions to meta. 2024-11-08T05:13:26,218 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T05:13:26,220 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731042806218"}]},"ts":"1731042806218"} 2024-11-08T05:13:26,225 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=test, state=ENABLING in hbase:meta 2024-11-08T05:13:26,226 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(204): Hosts are {d1ba675678c9=0} racks are {/default-rack=0} 2024-11-08T05:13:26,242 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-08T05:13:26,242 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-08T05:13:26,242 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-08T05:13:26,242 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-08T05:13:26,242 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-08T05:13:26,242 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-08T05:13:26,242 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-08T05:13:26,242 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-08T05:13:26,242 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-08T05:13:26,242 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-08T05:13:26,245 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=6240e38715889f3bb77bb89478c81a6c, ASSIGN}, {pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=eeaad0e7e7aa66f0b51c273c0aaf953c, ASSIGN}, {pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=68bbd0353f97ee12228f7798b0b987ea, ASSIGN}, {pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=64ddf686fed30cdb23b2e874efec598d, ASSIGN}, {pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d7680a9929f15640ca61551f186f33c1, ASSIGN}, {pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=65d5312dbd2f96bf3b16bdfd7504dff6, ASSIGN}, {pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=9b3274d80dcb7deed617c19ac1de4a68, ASSIGN}, {pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2d23e3e11ecc480e314e989d1bb8d0c3, ASSIGN}, {pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=fe1545b16a81758dcb9430dbf98c4ab9, ASSIGN}, {pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a12ea80f78b339cfbfc0e18b5338df0c, ASSIGN}, {pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=0e1c63297d3a8aea2fdbd1842301c811, ASSIGN}, {pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=40440b5f39bb5421276aa0b156481a9e, ASSIGN}, {pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=27c9aaf34aaf98cd5bc8030630ae7fe3, ASSIGN}, {pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=868527ec322eba1d7cbd6371701f7ad2, ASSIGN}, {pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d951febea78018bb9f10f07d8788575d, ASSIGN}, {pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=b0ebd9a9794acc56e430b36d1f17621c, ASSIGN}, {pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=7830dc6b1acea6e1d468f8008d8a82b9, ASSIGN}, {pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=12f1148a7b8db3e4f515337b8b7746fa, ASSIGN}, {pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=e26dcb2339b364f3699d110c15cd4bb3, ASSIGN}, {pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=445b14856951586918c91d4ba342f1ff, ASSIGN}, {pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=14efcc1b9822cc74f4d4a96648555167, ASSIGN}, {pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=7c8f72849b2e1719ed903afc73e0571a, ASSIGN}, {pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=81806cb184c70c9dc0df408694092d85, ASSIGN}, {pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=87b3455cffd1f4e06e5901b3bcbed9dc, ASSIGN}, {pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=de358f808110a48fa32e0533fefdc95c, ASSIGN}, {pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=e61dc9e059921fddc5ad7f6f7971acc6, ASSIGN}] 2024-11-08T05:13:26,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37587 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-08T05:13:26,253 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=e61dc9e059921fddc5ad7f6f7971acc6, ASSIGN 2024-11-08T05:13:26,255 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=e61dc9e059921fddc5ad7f6f7971acc6, ASSIGN; state=OFFLINE, location=d1ba675678c9,46753,1731042802851; forceNewPlan=false, retain=false 2024-11-08T05:13:26,256 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=de358f808110a48fa32e0533fefdc95c, ASSIGN 2024-11-08T05:13:26,257 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=de358f808110a48fa32e0533fefdc95c, ASSIGN; state=OFFLINE, location=d1ba675678c9,41485,1731042802970; forceNewPlan=false, retain=false 2024-11-08T05:13:26,258 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=87b3455cffd1f4e06e5901b3bcbed9dc, ASSIGN 2024-11-08T05:13:26,260 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=87b3455cffd1f4e06e5901b3bcbed9dc, ASSIGN; state=OFFLINE, location=d1ba675678c9,39291,1731042803020; forceNewPlan=false, retain=false 2024-11-08T05:13:26,261 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=81806cb184c70c9dc0df408694092d85, ASSIGN 2024-11-08T05:13:26,262 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=81806cb184c70c9dc0df408694092d85, ASSIGN; state=OFFLINE, location=d1ba675678c9,46753,1731042802851; forceNewPlan=false, retain=false 2024-11-08T05:13:26,264 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=7c8f72849b2e1719ed903afc73e0571a, ASSIGN 2024-11-08T05:13:26,265 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=7c8f72849b2e1719ed903afc73e0571a, ASSIGN; state=OFFLINE, location=d1ba675678c9,41485,1731042802970; forceNewPlan=false, retain=false 2024-11-08T05:13:26,266 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=14efcc1b9822cc74f4d4a96648555167, ASSIGN 2024-11-08T05:13:26,268 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=14efcc1b9822cc74f4d4a96648555167, ASSIGN; state=OFFLINE, location=d1ba675678c9,39291,1731042803020; forceNewPlan=false, retain=false 2024-11-08T05:13:26,269 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=445b14856951586918c91d4ba342f1ff, ASSIGN 2024-11-08T05:13:26,270 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=445b14856951586918c91d4ba342f1ff, ASSIGN; state=OFFLINE, location=d1ba675678c9,46753,1731042802851; forceNewPlan=false, retain=false 2024-11-08T05:13:26,271 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=e26dcb2339b364f3699d110c15cd4bb3, ASSIGN 2024-11-08T05:13:26,272 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=e26dcb2339b364f3699d110c15cd4bb3, ASSIGN; state=OFFLINE, location=d1ba675678c9,41485,1731042802970; forceNewPlan=false, retain=false 2024-11-08T05:13:26,274 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=12f1148a7b8db3e4f515337b8b7746fa, ASSIGN 2024-11-08T05:13:26,275 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=12f1148a7b8db3e4f515337b8b7746fa, ASSIGN; state=OFFLINE, location=d1ba675678c9,39291,1731042803020; forceNewPlan=false, retain=false 2024-11-08T05:13:26,276 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=7830dc6b1acea6e1d468f8008d8a82b9, ASSIGN 2024-11-08T05:13:26,277 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=7830dc6b1acea6e1d468f8008d8a82b9, ASSIGN; state=OFFLINE, location=d1ba675678c9,46753,1731042802851; forceNewPlan=false, retain=false 2024-11-08T05:13:26,278 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=b0ebd9a9794acc56e430b36d1f17621c, ASSIGN 2024-11-08T05:13:26,279 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=b0ebd9a9794acc56e430b36d1f17621c, ASSIGN; state=OFFLINE, location=d1ba675678c9,41485,1731042802970; forceNewPlan=false, retain=false 2024-11-08T05:13:26,281 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d951febea78018bb9f10f07d8788575d, ASSIGN 2024-11-08T05:13:26,282 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=d951febea78018bb9f10f07d8788575d, ASSIGN; state=OFFLINE, location=d1ba675678c9,39291,1731042803020; forceNewPlan=false, retain=false 2024-11-08T05:13:26,284 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=868527ec322eba1d7cbd6371701f7ad2, ASSIGN 2024-11-08T05:13:26,285 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=868527ec322eba1d7cbd6371701f7ad2, ASSIGN; state=OFFLINE, location=d1ba675678c9,46753,1731042802851; forceNewPlan=false, retain=false 2024-11-08T05:13:26,286 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=27c9aaf34aaf98cd5bc8030630ae7fe3, ASSIGN 2024-11-08T05:13:26,287 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=27c9aaf34aaf98cd5bc8030630ae7fe3, ASSIGN; state=OFFLINE, location=d1ba675678c9,41485,1731042802970; forceNewPlan=false, retain=false 2024-11-08T05:13:26,289 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=40440b5f39bb5421276aa0b156481a9e, ASSIGN 2024-11-08T05:13:26,290 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=40440b5f39bb5421276aa0b156481a9e, ASSIGN; state=OFFLINE, location=d1ba675678c9,39291,1731042803020; forceNewPlan=false, retain=false 2024-11-08T05:13:26,291 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=0e1c63297d3a8aea2fdbd1842301c811, ASSIGN 2024-11-08T05:13:26,293 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=0e1c63297d3a8aea2fdbd1842301c811, ASSIGN; state=OFFLINE, location=d1ba675678c9,41485,1731042802970; forceNewPlan=false, retain=false 2024-11-08T05:13:26,294 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a12ea80f78b339cfbfc0e18b5338df0c, ASSIGN 2024-11-08T05:13:26,295 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=a12ea80f78b339cfbfc0e18b5338df0c, ASSIGN; state=OFFLINE, location=d1ba675678c9,46753,1731042802851; forceNewPlan=false, retain=false 2024-11-08T05:13:26,296 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=fe1545b16a81758dcb9430dbf98c4ab9, ASSIGN 2024-11-08T05:13:26,298 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=fe1545b16a81758dcb9430dbf98c4ab9, ASSIGN; state=OFFLINE, location=d1ba675678c9,41485,1731042802970; forceNewPlan=false, retain=false 2024-11-08T05:13:26,299 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2d23e3e11ecc480e314e989d1bb8d0c3, ASSIGN 2024-11-08T05:13:26,300 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=2d23e3e11ecc480e314e989d1bb8d0c3, ASSIGN; state=OFFLINE, location=d1ba675678c9,39291,1731042803020; forceNewPlan=false, retain=false 2024-11-08T05:13:26,301 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=9b3274d80dcb7deed617c19ac1de4a68, ASSIGN 2024-11-08T05:13:26,302 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=9b3274d80dcb7deed617c19ac1de4a68, ASSIGN; state=OFFLINE, location=d1ba675678c9,46753,1731042802851; forceNewPlan=false, retain=false 2024-11-08T05:13:26,303 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=65d5312dbd2f96bf3b16bdfd7504dff6, ASSIGN 2024-11-08T05:13:26,304 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=65d5312dbd2f96bf3b16bdfd7504dff6, ASSIGN; state=OFFLINE, location=d1ba675678c9,41485,1731042802970; forceNewPlan=false, retain=false 2024-11-08T05:13:26,305 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d7680a9929f15640ca61551f186f33c1, ASSIGN 2024-11-08T05:13:26,306 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=d7680a9929f15640ca61551f186f33c1, ASSIGN; state=OFFLINE, location=d1ba675678c9,39291,1731042803020; forceNewPlan=false, retain=false 2024-11-08T05:13:26,308 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=64ddf686fed30cdb23b2e874efec598d, ASSIGN 2024-11-08T05:13:26,309 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=64ddf686fed30cdb23b2e874efec598d, ASSIGN; state=OFFLINE, location=d1ba675678c9,46753,1731042802851; forceNewPlan=false, retain=false 2024-11-08T05:13:26,311 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=68bbd0353f97ee12228f7798b0b987ea, ASSIGN 2024-11-08T05:13:26,312 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=68bbd0353f97ee12228f7798b0b987ea, ASSIGN; state=OFFLINE, location=d1ba675678c9,39291,1731042803020; forceNewPlan=false, retain=false 2024-11-08T05:13:26,313 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=eeaad0e7e7aa66f0b51c273c0aaf953c, ASSIGN 2024-11-08T05:13:26,315 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=eeaad0e7e7aa66f0b51c273c0aaf953c, ASSIGN; state=OFFLINE, location=d1ba675678c9,41485,1731042802970; forceNewPlan=false, retain=false 2024-11-08T05:13:26,316 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=6240e38715889f3bb77bb89478c81a6c, ASSIGN 2024-11-08T05:13:26,318 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=6240e38715889f3bb77bb89478c81a6c, ASSIGN; state=OFFLINE, location=d1ba675678c9,39291,1731042803020; forceNewPlan=false, retain=false 2024-11-08T05:13:26,408 INFO [d1ba675678c9:37587 {}] balancer.BaseLoadBalancer(388): Reassigned 26 regions. 26 retained the pre-restart assignment. 2024-11-08T05:13:26,409 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=e61dc9e059921fddc5ad7f6f7971acc6, regionState=OPENING, regionLocation=d1ba675678c9,46753,1731042802851 2024-11-08T05:13:26,410 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=81806cb184c70c9dc0df408694092d85, regionState=OPENING, regionLocation=d1ba675678c9,46753,1731042802851 2024-11-08T05:13:26,410 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=445b14856951586918c91d4ba342f1ff, regionState=OPENING, regionLocation=d1ba675678c9,46753,1731042802851 2024-11-08T05:13:26,411 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=7830dc6b1acea6e1d468f8008d8a82b9, regionState=OPENING, regionLocation=d1ba675678c9,46753,1731042802851 2024-11-08T05:13:26,411 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=868527ec322eba1d7cbd6371701f7ad2, regionState=OPENING, regionLocation=d1ba675678c9,46753,1731042802851 2024-11-08T05:13:26,412 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=a12ea80f78b339cfbfc0e18b5338df0c, regionState=OPENING, regionLocation=d1ba675678c9,46753,1731042802851 2024-11-08T05:13:26,412 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=9b3274d80dcb7deed617c19ac1de4a68, regionState=OPENING, regionLocation=d1ba675678c9,46753,1731042802851 2024-11-08T05:13:26,413 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=e61dc9e059921fddc5ad7f6f7971acc6, ASSIGN because future has completed 2024-11-08T05:13:26,413 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=64ddf686fed30cdb23b2e874efec598d, regionState=OPENING, regionLocation=d1ba675678c9,46753,1731042802851 2024-11-08T05:13:26,413 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=de358f808110a48fa32e0533fefdc95c, regionState=OPENING, regionLocation=d1ba675678c9,41485,1731042802970 2024-11-08T05:13:26,414 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure e61dc9e059921fddc5ad7f6f7971acc6, server=d1ba675678c9,46753,1731042802851}] 2024-11-08T05:13:26,415 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=81806cb184c70c9dc0df408694092d85, ASSIGN because future has completed 2024-11-08T05:13:26,417 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=445b14856951586918c91d4ba342f1ff, ASSIGN because future has completed 2024-11-08T05:13:26,419 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=7830dc6b1acea6e1d468f8008d8a82b9, ASSIGN because future has completed 2024-11-08T05:13:26,420 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7830dc6b1acea6e1d468f8008d8a82b9, server=d1ba675678c9,46753,1731042802851}] 2024-11-08T05:13:26,420 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=868527ec322eba1d7cbd6371701f7ad2, ASSIGN because future has completed 2024-11-08T05:13:26,423 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=a12ea80f78b339cfbfc0e18b5338df0c, ASSIGN because future has completed 2024-11-08T05:13:26,426 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=9b3274d80dcb7deed617c19ac1de4a68, ASSIGN because future has completed 2024-11-08T05:13:26,428 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=64ddf686fed30cdb23b2e874efec598d, ASSIGN because future has completed 2024-11-08T05:13:26,428 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9b3274d80dcb7deed617c19ac1de4a68, server=d1ba675678c9,46753,1731042802851}] 2024-11-08T05:13:26,429 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=de358f808110a48fa32e0533fefdc95c, ASSIGN because future has completed 2024-11-08T05:13:26,433 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure de358f808110a48fa32e0533fefdc95c, server=d1ba675678c9,41485,1731042802970}] 2024-11-08T05:13:26,438 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 64ddf686fed30cdb23b2e874efec598d, server=d1ba675678c9,46753,1731042802851}] 2024-11-08T05:13:26,444 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure a12ea80f78b339cfbfc0e18b5338df0c, server=d1ba675678c9,46753,1731042802851}] 2024-11-08T05:13:26,451 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure 868527ec322eba1d7cbd6371701f7ad2, server=d1ba675678c9,46753,1731042802851}] 2024-11-08T05:13:26,456 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure 445b14856951586918c91d4ba342f1ff, server=d1ba675678c9,46753,1731042802851}] 2024-11-08T05:13:26,460 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure 81806cb184c70c9dc0df408694092d85, server=d1ba675678c9,46753,1731042802851}] 2024-11-08T05:13:26,464 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=7c8f72849b2e1719ed903afc73e0571a, regionState=OPENING, regionLocation=d1ba675678c9,41485,1731042802970 2024-11-08T05:13:26,465 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=e26dcb2339b364f3699d110c15cd4bb3, regionState=OPENING, regionLocation=d1ba675678c9,41485,1731042802970 2024-11-08T05:13:26,465 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=b0ebd9a9794acc56e430b36d1f17621c, regionState=OPENING, regionLocation=d1ba675678c9,41485,1731042802970 2024-11-08T05:13:26,466 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=27c9aaf34aaf98cd5bc8030630ae7fe3, regionState=OPENING, regionLocation=d1ba675678c9,41485,1731042802970 2024-11-08T05:13:26,466 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=0e1c63297d3a8aea2fdbd1842301c811, regionState=OPENING, regionLocation=d1ba675678c9,41485,1731042802970 2024-11-08T05:13:26,467 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=fe1545b16a81758dcb9430dbf98c4ab9, regionState=OPENING, regionLocation=d1ba675678c9,41485,1731042802970 2024-11-08T05:13:26,467 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=65d5312dbd2f96bf3b16bdfd7504dff6, regionState=OPENING, regionLocation=d1ba675678c9,41485,1731042802970 2024-11-08T05:13:26,468 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=eeaad0e7e7aa66f0b51c273c0aaf953c, regionState=OPENING, regionLocation=d1ba675678c9,41485,1731042802970 2024-11-08T05:13:26,468 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=87b3455cffd1f4e06e5901b3bcbed9dc, regionState=OPENING, regionLocation=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:26,469 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=7c8f72849b2e1719ed903afc73e0571a, ASSIGN because future has completed 2024-11-08T05:13:26,469 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7c8f72849b2e1719ed903afc73e0571a, server=d1ba675678c9,41485,1731042802970}] 2024-11-08T05:13:26,471 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=e26dcb2339b364f3699d110c15cd4bb3, ASSIGN because future has completed 2024-11-08T05:13:26,472 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=b0ebd9a9794acc56e430b36d1f17621c, ASSIGN because future has completed 2024-11-08T05:13:26,475 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=27c9aaf34aaf98cd5bc8030630ae7fe3, ASSIGN because future has completed 2024-11-08T05:13:26,476 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure 27c9aaf34aaf98cd5bc8030630ae7fe3, server=d1ba675678c9,41485,1731042802970}] 2024-11-08T05:13:26,478 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=0e1c63297d3a8aea2fdbd1842301c811, ASSIGN because future has completed 2024-11-08T05:13:26,480 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=fe1545b16a81758dcb9430dbf98c4ab9, ASSIGN because future has completed 2024-11-08T05:13:26,482 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=65d5312dbd2f96bf3b16bdfd7504dff6, ASSIGN because future has completed 2024-11-08T05:13:26,482 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=42, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 65d5312dbd2f96bf3b16bdfd7504dff6, server=d1ba675678c9,41485,1731042802970}] 2024-11-08T05:13:26,483 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=eeaad0e7e7aa66f0b51c273c0aaf953c, ASSIGN because future has completed 2024-11-08T05:13:26,485 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=87b3455cffd1f4e06e5901b3bcbed9dc, ASSIGN because future has completed 2024-11-08T05:13:26,488 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure 87b3455cffd1f4e06e5901b3bcbed9dc, server=d1ba675678c9,39291,1731042803020}] 2024-11-08T05:13:26,493 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=44, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure eeaad0e7e7aa66f0b51c273c0aaf953c, server=d1ba675678c9,41485,1731042802970}] 2024-11-08T05:13:26,499 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure fe1545b16a81758dcb9430dbf98c4ab9, server=d1ba675678c9,41485,1731042802970}] 2024-11-08T05:13:26,504 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=46, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0e1c63297d3a8aea2fdbd1842301c811, server=d1ba675678c9,41485,1731042802970}] 2024-11-08T05:13:26,509 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure b0ebd9a9794acc56e430b36d1f17621c, server=d1ba675678c9,41485,1731042802970}] 2024-11-08T05:13:26,515 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=48, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure e26dcb2339b364f3699d110c15cd4bb3, server=d1ba675678c9,41485,1731042802970}] 2024-11-08T05:13:26,519 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=14efcc1b9822cc74f4d4a96648555167, regionState=OPENING, regionLocation=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:26,520 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=12f1148a7b8db3e4f515337b8b7746fa, regionState=OPENING, regionLocation=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:26,521 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=d951febea78018bb9f10f07d8788575d, regionState=OPENING, regionLocation=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:26,521 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=40440b5f39bb5421276aa0b156481a9e, regionState=OPENING, regionLocation=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:26,522 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=2d23e3e11ecc480e314e989d1bb8d0c3, regionState=OPENING, regionLocation=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:26,523 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=14efcc1b9822cc74f4d4a96648555167, ASSIGN because future has completed 2024-11-08T05:13:26,523 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure 14efcc1b9822cc74f4d4a96648555167, server=d1ba675678c9,39291,1731042803020}] 2024-11-08T05:13:26,526 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=12f1148a7b8db3e4f515337b8b7746fa, ASSIGN because future has completed 2024-11-08T05:13:26,527 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=d951febea78018bb9f10f07d8788575d, ASSIGN because future has completed 2024-11-08T05:13:26,529 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=40440b5f39bb5421276aa0b156481a9e, ASSIGN because future has completed 2024-11-08T05:13:26,530 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=50, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 40440b5f39bb5421276aa0b156481a9e, server=d1ba675678c9,39291,1731042803020}] 2024-11-08T05:13:26,530 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=2d23e3e11ecc480e314e989d1bb8d0c3, ASSIGN because future has completed 2024-11-08T05:13:26,535 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2d23e3e11ecc480e314e989d1bb8d0c3, server=d1ba675678c9,39291,1731042803020}] 2024-11-08T05:13:26,540 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=52, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure d951febea78018bb9f10f07d8788575d, server=d1ba675678c9,39291,1731042803020}] 2024-11-08T05:13:26,545 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure 12f1148a7b8db3e4f515337b8b7746fa, server=d1ba675678c9,39291,1731042803020}] 2024-11-08T05:13:26,551 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=d7680a9929f15640ca61551f186f33c1, regionState=OPENING, regionLocation=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:26,552 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=68bbd0353f97ee12228f7798b0b987ea, regionState=OPENING, regionLocation=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:26,553 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=6240e38715889f3bb77bb89478c81a6c, regionState=OPENING, regionLocation=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:26,555 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=d7680a9929f15640ca61551f186f33c1, ASSIGN because future has completed 2024-11-08T05:13:26,555 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=54, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure d7680a9929f15640ca61551f186f33c1, server=d1ba675678c9,39291,1731042803020}] 2024-11-08T05:13:26,557 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=68bbd0353f97ee12228f7798b0b987ea, ASSIGN because future has completed 2024-11-08T05:13:26,558 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=6240e38715889f3bb77bb89478c81a6c, ASSIGN because future has completed 2024-11-08T05:13:26,560 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6240e38715889f3bb77bb89478c81a6c, server=d1ba675678c9,39291,1731042803020}] 2024-11-08T05:13:26,564 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 68bbd0353f97ee12228f7798b0b987ea, server=d1ba675678c9,39291,1731042803020}] 2024-11-08T05:13:26,573 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(132): Open test,qqq,1731042805591.7830dc6b1acea6e1d468f8008d8a82b9. 2024-11-08T05:13:26,574 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7752): Opening region: {ENCODED => 7830dc6b1acea6e1d468f8008d8a82b9, NAME => 'test,qqq,1731042805591.7830dc6b1acea6e1d468f8008d8a82b9.', STARTKEY => 'qqq', ENDKEY => 'rrr'} 2024-11-08T05:13:26,574 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 7830dc6b1acea6e1d468f8008d8a82b9 2024-11-08T05:13:26,574 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(898): Instantiated test,qqq,1731042805591.7830dc6b1acea6e1d468f8008d8a82b9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T05:13:26,574 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7794): checking encryption for 7830dc6b1acea6e1d468f8008d8a82b9 2024-11-08T05:13:26,574 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7797): checking classloading for 7830dc6b1acea6e1d468f8008d8a82b9 2024-11-08T05:13:26,577 INFO [StoreOpener-7830dc6b1acea6e1d468f8008d8a82b9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 7830dc6b1acea6e1d468f8008d8a82b9 2024-11-08T05:13:26,579 INFO [StoreOpener-7830dc6b1acea6e1d468f8008d8a82b9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7830dc6b1acea6e1d468f8008d8a82b9 columnFamilyName family 2024-11-08T05:13:26,579 DEBUG [StoreOpener-7830dc6b1acea6e1d468f8008d8a82b9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T05:13:26,580 INFO [StoreOpener-7830dc6b1acea6e1d468f8008d8a82b9-1 {}] regionserver.HStore(327): Store=7830dc6b1acea6e1d468f8008d8a82b9/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T05:13:26,580 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1038): replaying wal for 7830dc6b1acea6e1d468f8008d8a82b9 2024-11-08T05:13:26,582 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/default/test/7830dc6b1acea6e1d468f8008d8a82b9 2024-11-08T05:13:26,583 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/default/test/7830dc6b1acea6e1d468f8008d8a82b9 2024-11-08T05:13:26,583 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1048): stopping wal replay for 7830dc6b1acea6e1d468f8008d8a82b9 2024-11-08T05:13:26,583 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1060): Cleaning up temporary data for 7830dc6b1acea6e1d468f8008d8a82b9 2024-11-08T05:13:26,586 DEBUG [RSProcedureDispatcher-pool-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-08T05:13:26,586 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1093): writing seq id for 7830dc6b1acea6e1d468f8008d8a82b9 2024-11-08T05:13:26,590 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45391, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-08T05:13:26,590 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/default/test/7830dc6b1acea6e1d468f8008d8a82b9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T05:13:26,591 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1114): Opened 7830dc6b1acea6e1d468f8008d8a82b9; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72061540, jitterRate=0.07380062341690063}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T05:13:26,591 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7830dc6b1acea6e1d468f8008d8a82b9 2024-11-08T05:13:26,592 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1006): Region open journal for 7830dc6b1acea6e1d468f8008d8a82b9: Running coprocessor pre-open hook at 1731042806574Writing region info on filesystem at 1731042806574Initializing all the Stores at 1731042806576 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731042806576Cleaning up temporary data from old regions at 1731042806584 (+8 ms)Running coprocessor post-open hooks at 1731042806591 (+7 ms)Region opened successfully at 1731042806592 (+1 ms) 2024-11-08T05:13:26,594 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(132): Open test,iii,1731042805591.fe1545b16a81758dcb9430dbf98c4ab9. 2024-11-08T05:13:26,594 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2236): Post open deploy tasks for test,qqq,1731042805591.7830dc6b1acea6e1d468f8008d8a82b9., pid=32, masterSystemTime=1731042806567 2024-11-08T05:13:26,595 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7752): Opening region: {ENCODED => fe1545b16a81758dcb9430dbf98c4ab9, NAME => 'test,iii,1731042805591.fe1545b16a81758dcb9430dbf98c4ab9.', STARTKEY => 'iii', ENDKEY => 'jjj'} 2024-11-08T05:13:26,595 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test fe1545b16a81758dcb9430dbf98c4ab9 2024-11-08T05:13:26,595 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(898): Instantiated test,iii,1731042805591.fe1545b16a81758dcb9430dbf98c4ab9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T05:13:26,595 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7794): checking encryption for fe1545b16a81758dcb9430dbf98c4ab9 2024-11-08T05:13:26,595 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7797): checking classloading for fe1545b16a81758dcb9430dbf98c4ab9 2024-11-08T05:13:26,598 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2266): Finished post open deploy task for test,qqq,1731042805591.7830dc6b1acea6e1d468f8008d8a82b9. 2024-11-08T05:13:26,598 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(153): Opened test,qqq,1731042805591.7830dc6b1acea6e1d468f8008d8a82b9. 2024-11-08T05:13:26,598 INFO [StoreOpener-fe1545b16a81758dcb9430dbf98c4ab9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region fe1545b16a81758dcb9430dbf98c4ab9 2024-11-08T05:13:26,598 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(132): Open test,nnn,1731042805591.868527ec322eba1d7cbd6371701f7ad2. 2024-11-08T05:13:26,598 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7752): Opening region: {ENCODED => 868527ec322eba1d7cbd6371701f7ad2, NAME => 'test,nnn,1731042805591.868527ec322eba1d7cbd6371701f7ad2.', STARTKEY => 'nnn', ENDKEY => 'ooo'} 2024-11-08T05:13:26,599 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=7830dc6b1acea6e1d468f8008d8a82b9, regionState=OPEN, openSeqNum=2, regionLocation=d1ba675678c9,46753,1731042802851 2024-11-08T05:13:26,599 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 868527ec322eba1d7cbd6371701f7ad2 2024-11-08T05:13:26,599 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(898): Instantiated test,nnn,1731042805591.868527ec322eba1d7cbd6371701f7ad2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T05:13:26,599 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7794): checking encryption for 868527ec322eba1d7cbd6371701f7ad2 2024-11-08T05:13:26,599 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7797): checking classloading for 868527ec322eba1d7cbd6371701f7ad2 2024-11-08T05:13:26,600 INFO [StoreOpener-fe1545b16a81758dcb9430dbf98c4ab9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fe1545b16a81758dcb9430dbf98c4ab9 columnFamilyName family 2024-11-08T05:13:26,600 DEBUG [StoreOpener-fe1545b16a81758dcb9430dbf98c4ab9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T05:13:26,601 INFO [StoreOpener-fe1545b16a81758dcb9430dbf98c4ab9-1 {}] regionserver.HStore(327): Store=fe1545b16a81758dcb9430dbf98c4ab9/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T05:13:26,601 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1038): replaying wal for fe1545b16a81758dcb9430dbf98c4ab9 2024-11-08T05:13:26,602 INFO [StoreOpener-868527ec322eba1d7cbd6371701f7ad2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 868527ec322eba1d7cbd6371701f7ad2 2024-11-08T05:13:26,602 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7830dc6b1acea6e1d468f8008d8a82b9, server=d1ba675678c9,46753,1731042802851 because future has completed 2024-11-08T05:13:26,602 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/default/test/fe1545b16a81758dcb9430dbf98c4ab9 2024-11-08T05:13:26,603 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/default/test/fe1545b16a81758dcb9430dbf98c4ab9 2024-11-08T05:13:26,604 INFO [StoreOpener-868527ec322eba1d7cbd6371701f7ad2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 868527ec322eba1d7cbd6371701f7ad2 columnFamilyName family 2024-11-08T05:13:26,604 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1048): stopping wal replay for fe1545b16a81758dcb9430dbf98c4ab9 2024-11-08T05:13:26,604 DEBUG [StoreOpener-868527ec322eba1d7cbd6371701f7ad2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T05:13:26,604 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1060): Cleaning up temporary data for fe1545b16a81758dcb9430dbf98c4ab9 2024-11-08T05:13:26,604 INFO [StoreOpener-868527ec322eba1d7cbd6371701f7ad2-1 {}] regionserver.HStore(327): Store=868527ec322eba1d7cbd6371701f7ad2/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T05:13:26,605 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1038): replaying wal for 868527ec322eba1d7cbd6371701f7ad2 2024-11-08T05:13:26,607 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/default/test/868527ec322eba1d7cbd6371701f7ad2 2024-11-08T05:13:26,607 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=21 2024-11-08T05:13:26,607 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=21, state=SUCCESS, hasLock=false; OpenRegionProcedure 7830dc6b1acea6e1d468f8008d8a82b9, server=d1ba675678c9,46753,1731042802851 in 184 msec 2024-11-08T05:13:26,607 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/default/test/868527ec322eba1d7cbd6371701f7ad2 2024-11-08T05:13:26,608 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1048): stopping wal replay for 868527ec322eba1d7cbd6371701f7ad2 2024-11-08T05:13:26,608 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1060): Cleaning up temporary data for 868527ec322eba1d7cbd6371701f7ad2 2024-11-08T05:13:26,608 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1093): writing seq id for fe1545b16a81758dcb9430dbf98c4ab9 2024-11-08T05:13:26,610 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=7830dc6b1acea6e1d468f8008d8a82b9, ASSIGN in 363 msec 2024-11-08T05:13:26,611 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1093): writing seq id for 868527ec322eba1d7cbd6371701f7ad2 2024-11-08T05:13:26,611 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/default/test/fe1545b16a81758dcb9430dbf98c4ab9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T05:13:26,612 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1114): Opened fe1545b16a81758dcb9430dbf98c4ab9; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64878500, jitterRate=-0.033235013484954834}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T05:13:26,612 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fe1545b16a81758dcb9430dbf98c4ab9 2024-11-08T05:13:26,612 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1006): Region open journal for fe1545b16a81758dcb9430dbf98c4ab9: Running coprocessor pre-open hook at 1731042806596Writing region info on filesystem at 1731042806596Initializing all the Stores at 1731042806597 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731042806597Cleaning up temporary data from old regions at 1731042806604 (+7 ms)Running coprocessor post-open hooks at 1731042806612 (+8 ms)Region opened successfully at 1731042806612 2024-11-08T05:13:26,614 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2236): Post open deploy tasks for test,iii,1731042805591.fe1545b16a81758dcb9430dbf98c4ab9., pid=45, masterSystemTime=1731042806586 2024-11-08T05:13:26,617 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2266): Finished post open deploy task for test,iii,1731042805591.fe1545b16a81758dcb9430dbf98c4ab9. 2024-11-08T05:13:26,617 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(153): Opened test,iii,1731042805591.fe1545b16a81758dcb9430dbf98c4ab9. 2024-11-08T05:13:26,617 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/default/test/868527ec322eba1d7cbd6371701f7ad2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T05:13:26,617 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(132): Open test,fff,1731042805591.65d5312dbd2f96bf3b16bdfd7504dff6. 2024-11-08T05:13:26,617 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7752): Opening region: {ENCODED => 65d5312dbd2f96bf3b16bdfd7504dff6, NAME => 'test,fff,1731042805591.65d5312dbd2f96bf3b16bdfd7504dff6.', STARTKEY => 'fff', ENDKEY => 'ggg'} 2024-11-08T05:13:26,618 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 65d5312dbd2f96bf3b16bdfd7504dff6 2024-11-08T05:13:26,618 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(898): Instantiated test,fff,1731042805591.65d5312dbd2f96bf3b16bdfd7504dff6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T05:13:26,618 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=fe1545b16a81758dcb9430dbf98c4ab9, regionState=OPEN, openSeqNum=2, regionLocation=d1ba675678c9,41485,1731042802970 2024-11-08T05:13:26,618 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7794): checking encryption for 65d5312dbd2f96bf3b16bdfd7504dff6 2024-11-08T05:13:26,618 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1114): Opened 868527ec322eba1d7cbd6371701f7ad2; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75058751, jitterRate=0.11846254765987396}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T05:13:26,618 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7797): checking classloading for 65d5312dbd2f96bf3b16bdfd7504dff6 2024-11-08T05:13:26,618 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 868527ec322eba1d7cbd6371701f7ad2 2024-11-08T05:13:26,618 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1006): Region open journal for 868527ec322eba1d7cbd6371701f7ad2: Running coprocessor pre-open hook at 1731042806599Writing region info on filesystem at 1731042806599Initializing all the Stores at 1731042806601 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731042806601Cleaning up temporary data from old regions at 1731042806608 (+7 ms)Running coprocessor post-open hooks at 1731042806618 (+10 ms)Region opened successfully at 1731042806618 2024-11-08T05:13:26,620 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2236): Post open deploy tasks for test,nnn,1731042805591.868527ec322eba1d7cbd6371701f7ad2., pid=37, masterSystemTime=1731042806567 2024-11-08T05:13:26,621 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=45, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure fe1545b16a81758dcb9430dbf98c4ab9, server=d1ba675678c9,41485,1731042802970 because future has completed 2024-11-08T05:13:26,621 INFO [StoreOpener-65d5312dbd2f96bf3b16bdfd7504dff6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 65d5312dbd2f96bf3b16bdfd7504dff6 2024-11-08T05:13:26,624 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2266): Finished post open deploy task for test,nnn,1731042805591.868527ec322eba1d7cbd6371701f7ad2. 2024-11-08T05:13:26,624 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(153): Opened test,nnn,1731042805591.868527ec322eba1d7cbd6371701f7ad2. 2024-11-08T05:13:26,624 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(132): Open test,zzz,1731042805591.e61dc9e059921fddc5ad7f6f7971acc6. 2024-11-08T05:13:26,624 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7752): Opening region: {ENCODED => e61dc9e059921fddc5ad7f6f7971acc6, NAME => 'test,zzz,1731042805591.e61dc9e059921fddc5ad7f6f7971acc6.', STARTKEY => 'zzz', ENDKEY => ''} 2024-11-08T05:13:26,625 INFO [StoreOpener-65d5312dbd2f96bf3b16bdfd7504dff6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 65d5312dbd2f96bf3b16bdfd7504dff6 columnFamilyName family 2024-11-08T05:13:26,625 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test e61dc9e059921fddc5ad7f6f7971acc6 2024-11-08T05:13:26,625 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(898): Instantiated test,zzz,1731042805591.e61dc9e059921fddc5ad7f6f7971acc6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T05:13:26,625 DEBUG [StoreOpener-65d5312dbd2f96bf3b16bdfd7504dff6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T05:13:26,625 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7794): checking encryption for e61dc9e059921fddc5ad7f6f7971acc6 2024-11-08T05:13:26,625 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7797): checking classloading for e61dc9e059921fddc5ad7f6f7971acc6 2024-11-08T05:13:26,626 INFO [StoreOpener-65d5312dbd2f96bf3b16bdfd7504dff6-1 {}] regionserver.HStore(327): Store=65d5312dbd2f96bf3b16bdfd7504dff6/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T05:13:26,627 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1038): replaying wal for 65d5312dbd2f96bf3b16bdfd7504dff6 2024-11-08T05:13:26,629 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/default/test/65d5312dbd2f96bf3b16bdfd7504dff6 2024-11-08T05:13:26,629 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=13 2024-11-08T05:13:26,629 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=13, state=SUCCESS, hasLock=false; OpenRegionProcedure fe1545b16a81758dcb9430dbf98c4ab9, server=d1ba675678c9,41485,1731042802970 in 124 msec 2024-11-08T05:13:26,630 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/default/test/65d5312dbd2f96bf3b16bdfd7504dff6 2024-11-08T05:13:26,630 INFO [StoreOpener-e61dc9e059921fddc5ad7f6f7971acc6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region e61dc9e059921fddc5ad7f6f7971acc6 2024-11-08T05:13:26,631 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1048): stopping wal replay for 65d5312dbd2f96bf3b16bdfd7504dff6 2024-11-08T05:13:26,631 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1060): Cleaning up temporary data for 65d5312dbd2f96bf3b16bdfd7504dff6 2024-11-08T05:13:26,632 INFO [StoreOpener-e61dc9e059921fddc5ad7f6f7971acc6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e61dc9e059921fddc5ad7f6f7971acc6 columnFamilyName family 2024-11-08T05:13:26,633 DEBUG [StoreOpener-e61dc9e059921fddc5ad7f6f7971acc6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T05:13:26,633 INFO [StoreOpener-e61dc9e059921fddc5ad7f6f7971acc6-1 {}] regionserver.HStore(327): Store=e61dc9e059921fddc5ad7f6f7971acc6/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T05:13:26,634 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=fe1545b16a81758dcb9430dbf98c4ab9, ASSIGN in 385 msec 2024-11-08T05:13:26,634 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1038): replaying wal for e61dc9e059921fddc5ad7f6f7971acc6 2024-11-08T05:13:26,634 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1093): writing seq id for 65d5312dbd2f96bf3b16bdfd7504dff6 2024-11-08T05:13:26,635 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/default/test/e61dc9e059921fddc5ad7f6f7971acc6 2024-11-08T05:13:26,635 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/default/test/e61dc9e059921fddc5ad7f6f7971acc6 2024-11-08T05:13:26,636 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=868527ec322eba1d7cbd6371701f7ad2, regionState=OPEN, openSeqNum=2, regionLocation=d1ba675678c9,46753,1731042802851 2024-11-08T05:13:26,636 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1048): stopping wal replay for e61dc9e059921fddc5ad7f6f7971acc6 2024-11-08T05:13:26,636 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1060): Cleaning up temporary data for e61dc9e059921fddc5ad7f6f7971acc6 2024-11-08T05:13:26,640 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure 868527ec322eba1d7cbd6371701f7ad2, server=d1ba675678c9,46753,1731042802851 because future has completed 2024-11-08T05:13:26,640 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1093): writing seq id for e61dc9e059921fddc5ad7f6f7971acc6 2024-11-08T05:13:26,641 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/default/test/65d5312dbd2f96bf3b16bdfd7504dff6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T05:13:26,641 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-08T05:13:26,642 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1114): Opened 65d5312dbd2f96bf3b16bdfd7504dff6; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68202648, jitterRate=0.016298651695251465}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T05:13:26,642 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 65d5312dbd2f96bf3b16bdfd7504dff6 2024-11-08T05:13:26,642 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1006): Region open journal for 65d5312dbd2f96bf3b16bdfd7504dff6: Running coprocessor pre-open hook at 1731042806618Writing region info on filesystem at 1731042806618Initializing all the Stores at 1731042806620 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731042806620Cleaning up temporary data from old regions at 1731042806631 (+11 ms)Running coprocessor post-open hooks at 1731042806642 (+11 ms)Region opened successfully at 1731042806642 2024-11-08T05:13:26,643 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60257, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-08T05:13:26,644 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2236): Post open deploy tasks for test,fff,1731042805591.65d5312dbd2f96bf3b16bdfd7504dff6., pid=42, masterSystemTime=1731042806586 2024-11-08T05:13:26,646 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/default/test/e61dc9e059921fddc5ad7f6f7971acc6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T05:13:26,647 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1114): Opened e61dc9e059921fddc5ad7f6f7971acc6; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69757492, jitterRate=0.03946763277053833}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T05:13:26,648 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e61dc9e059921fddc5ad7f6f7971acc6 2024-11-08T05:13:26,648 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1006): Region open journal for e61dc9e059921fddc5ad7f6f7971acc6: Running coprocessor pre-open hook at 1731042806625Writing region info on filesystem at 1731042806625Initializing all the Stores at 1731042806630 (+5 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731042806630Cleaning up temporary data from old regions at 1731042806636 (+6 ms)Running coprocessor post-open hooks at 1731042806648 (+12 ms)Region opened successfully at 1731042806648 2024-11-08T05:13:26,649 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2266): Finished post open deploy task for test,fff,1731042805591.65d5312dbd2f96bf3b16bdfd7504dff6. 2024-11-08T05:13:26,649 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(153): Opened test,fff,1731042805591.65d5312dbd2f96bf3b16bdfd7504dff6. 2024-11-08T05:13:26,649 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(132): Open test,sss,1731042805591.e26dcb2339b364f3699d110c15cd4bb3. 2024-11-08T05:13:26,649 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2236): Post open deploy tasks for test,zzz,1731042805591.e61dc9e059921fddc5ad7f6f7971acc6., pid=31, masterSystemTime=1731042806567 2024-11-08T05:13:26,649 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7752): Opening region: {ENCODED => e26dcb2339b364f3699d110c15cd4bb3, NAME => 'test,sss,1731042805591.e26dcb2339b364f3699d110c15cd4bb3.', STARTKEY => 'sss', ENDKEY => 'ttt'} 2024-11-08T05:13:26,649 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test e26dcb2339b364f3699d110c15cd4bb3 2024-11-08T05:13:26,650 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(898): Instantiated test,sss,1731042805591.e26dcb2339b364f3699d110c15cd4bb3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T05:13:26,650 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7794): checking encryption for e26dcb2339b364f3699d110c15cd4bb3 2024-11-08T05:13:26,650 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7797): checking classloading for e26dcb2339b364f3699d110c15cd4bb3 2024-11-08T05:13:26,652 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(132): Open test,rrr,1731042805591.12f1148a7b8db3e4f515337b8b7746fa. 2024-11-08T05:13:26,653 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7752): Opening region: {ENCODED => 12f1148a7b8db3e4f515337b8b7746fa, NAME => 'test,rrr,1731042805591.12f1148a7b8db3e4f515337b8b7746fa.', STARTKEY => 'rrr', ENDKEY => 'sss'} 2024-11-08T05:13:26,653 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 12f1148a7b8db3e4f515337b8b7746fa 2024-11-08T05:13:26,653 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(898): Instantiated test,rrr,1731042805591.12f1148a7b8db3e4f515337b8b7746fa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T05:13:26,654 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7794): checking encryption for 12f1148a7b8db3e4f515337b8b7746fa 2024-11-08T05:13:26,654 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7797): checking classloading for 12f1148a7b8db3e4f515337b8b7746fa 2024-11-08T05:13:26,656 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=18 2024-11-08T05:13:26,656 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=18, state=SUCCESS, hasLock=false; OpenRegionProcedure 868527ec322eba1d7cbd6371701f7ad2, server=d1ba675678c9,46753,1731042802851 in 193 msec 2024-11-08T05:13:26,656 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2266): Finished post open deploy task for test,zzz,1731042805591.e61dc9e059921fddc5ad7f6f7971acc6. 2024-11-08T05:13:26,656 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(153): Opened test,zzz,1731042805591.e61dc9e059921fddc5ad7f6f7971acc6. 2024-11-08T05:13:26,657 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(132): Open test,jjj,1731042805591.a12ea80f78b339cfbfc0e18b5338df0c. 2024-11-08T05:13:26,657 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7752): Opening region: {ENCODED => a12ea80f78b339cfbfc0e18b5338df0c, NAME => 'test,jjj,1731042805591.a12ea80f78b339cfbfc0e18b5338df0c.', STARTKEY => 'jjj', ENDKEY => 'kkk'} 2024-11-08T05:13:26,657 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test a12ea80f78b339cfbfc0e18b5338df0c 2024-11-08T05:13:26,657 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(898): Instantiated test,jjj,1731042805591.a12ea80f78b339cfbfc0e18b5338df0c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T05:13:26,658 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=e61dc9e059921fddc5ad7f6f7971acc6, regionState=OPEN, openSeqNum=2, regionLocation=d1ba675678c9,46753,1731042802851 2024-11-08T05:13:26,658 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7794): checking encryption for a12ea80f78b339cfbfc0e18b5338df0c 2024-11-08T05:13:26,658 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7797): checking classloading for a12ea80f78b339cfbfc0e18b5338df0c 2024-11-08T05:13:26,661 INFO [StoreOpener-a12ea80f78b339cfbfc0e18b5338df0c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region a12ea80f78b339cfbfc0e18b5338df0c 2024-11-08T05:13:26,662 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure e61dc9e059921fddc5ad7f6f7971acc6, server=d1ba675678c9,46753,1731042802851 because future has completed 2024-11-08T05:13:26,662 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=868527ec322eba1d7cbd6371701f7ad2, ASSIGN in 416 msec 2024-11-08T05:13:26,663 INFO [StoreOpener-a12ea80f78b339cfbfc0e18b5338df0c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a12ea80f78b339cfbfc0e18b5338df0c columnFamilyName family 2024-11-08T05:13:26,663 DEBUG [StoreOpener-a12ea80f78b339cfbfc0e18b5338df0c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T05:13:26,664 INFO [StoreOpener-a12ea80f78b339cfbfc0e18b5338df0c-1 {}] regionserver.HStore(327): Store=a12ea80f78b339cfbfc0e18b5338df0c/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T05:13:26,664 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1038): replaying wal for a12ea80f78b339cfbfc0e18b5338df0c 2024-11-08T05:13:26,665 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/default/test/a12ea80f78b339cfbfc0e18b5338df0c 2024-11-08T05:13:26,666 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/default/test/a12ea80f78b339cfbfc0e18b5338df0c 2024-11-08T05:13:26,667 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1048): stopping wal replay for a12ea80f78b339cfbfc0e18b5338df0c 2024-11-08T05:13:26,667 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1060): Cleaning up temporary data for a12ea80f78b339cfbfc0e18b5338df0c 2024-11-08T05:13:26,670 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1093): writing seq id for a12ea80f78b339cfbfc0e18b5338df0c 2024-11-08T05:13:26,671 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-11-08T05:13:26,671 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; OpenRegionProcedure e61dc9e059921fddc5ad7f6f7971acc6, server=d1ba675678c9,46753,1731042802851 in 250 msec 2024-11-08T05:13:26,674 INFO [StoreOpener-e26dcb2339b364f3699d110c15cd4bb3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region e26dcb2339b364f3699d110c15cd4bb3 2024-11-08T05:13:26,676 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/default/test/a12ea80f78b339cfbfc0e18b5338df0c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T05:13:26,676 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=e61dc9e059921fddc5ad7f6f7971acc6, ASSIGN in 427 msec 2024-11-08T05:13:26,677 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1114): Opened a12ea80f78b339cfbfc0e18b5338df0c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62982931, jitterRate=-0.06148119270801544}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T05:13:26,677 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a12ea80f78b339cfbfc0e18b5338df0c 2024-11-08T05:13:26,677 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1006): Region open journal for a12ea80f78b339cfbfc0e18b5338df0c: Running coprocessor pre-open hook at 1731042806658Writing region info on filesystem at 1731042806658Initializing all the Stores at 1731042806660 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731042806660Cleaning up temporary data from old regions at 1731042806667 (+7 ms)Running coprocessor post-open hooks at 1731042806677 (+10 ms)Region opened successfully at 1731042806677 2024-11-08T05:13:26,679 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2236): Post open deploy tasks for test,jjj,1731042805591.a12ea80f78b339cfbfc0e18b5338df0c., pid=36, masterSystemTime=1731042806567 2024-11-08T05:13:26,679 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=65d5312dbd2f96bf3b16bdfd7504dff6, regionState=OPEN, openSeqNum=2, regionLocation=d1ba675678c9,41485,1731042802970 2024-11-08T05:13:26,682 INFO [StoreOpener-e26dcb2339b364f3699d110c15cd4bb3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e26dcb2339b364f3699d110c15cd4bb3 columnFamilyName family 2024-11-08T05:13:26,682 DEBUG [StoreOpener-e26dcb2339b364f3699d110c15cd4bb3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T05:13:26,683 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=42, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 65d5312dbd2f96bf3b16bdfd7504dff6, server=d1ba675678c9,41485,1731042802970 because future has completed 2024-11-08T05:13:26,683 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2266): Finished post open deploy task for test,jjj,1731042805591.a12ea80f78b339cfbfc0e18b5338df0c. 2024-11-08T05:13:26,683 INFO [StoreOpener-e26dcb2339b364f3699d110c15cd4bb3-1 {}] regionserver.HStore(327): Store=e26dcb2339b364f3699d110c15cd4bb3/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T05:13:26,683 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(153): Opened test,jjj,1731042805591.a12ea80f78b339cfbfc0e18b5338df0c. 2024-11-08T05:13:26,683 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1038): replaying wal for e26dcb2339b364f3699d110c15cd4bb3 2024-11-08T05:13:26,683 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(132): Open test,ddd,1731042805591.64ddf686fed30cdb23b2e874efec598d. 2024-11-08T05:13:26,684 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7752): Opening region: {ENCODED => 64ddf686fed30cdb23b2e874efec598d, NAME => 'test,ddd,1731042805591.64ddf686fed30cdb23b2e874efec598d.', STARTKEY => 'ddd', ENDKEY => 'eee'} 2024-11-08T05:13:26,684 INFO [StoreOpener-12f1148a7b8db3e4f515337b8b7746fa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 12f1148a7b8db3e4f515337b8b7746fa 2024-11-08T05:13:26,684 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 64ddf686fed30cdb23b2e874efec598d 2024-11-08T05:13:26,684 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(898): Instantiated test,ddd,1731042805591.64ddf686fed30cdb23b2e874efec598d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T05:13:26,684 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7794): checking encryption for 64ddf686fed30cdb23b2e874efec598d 2024-11-08T05:13:26,685 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=a12ea80f78b339cfbfc0e18b5338df0c, regionState=OPEN, openSeqNum=2, regionLocation=d1ba675678c9,46753,1731042802851 2024-11-08T05:13:26,685 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7797): checking classloading for 64ddf686fed30cdb23b2e874efec598d 2024-11-08T05:13:26,685 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/default/test/e26dcb2339b364f3699d110c15cd4bb3 2024-11-08T05:13:26,687 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/default/test/e26dcb2339b364f3699d110c15cd4bb3 2024-11-08T05:13:26,687 INFO [StoreOpener-12f1148a7b8db3e4f515337b8b7746fa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 12f1148a7b8db3e4f515337b8b7746fa columnFamilyName family 2024-11-08T05:13:26,688 DEBUG [StoreOpener-12f1148a7b8db3e4f515337b8b7746fa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T05:13:26,688 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1048): stopping wal replay for e26dcb2339b364f3699d110c15cd4bb3 2024-11-08T05:13:26,688 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure a12ea80f78b339cfbfc0e18b5338df0c, server=d1ba675678c9,46753,1731042802851 because future has completed 2024-11-08T05:13:26,688 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1060): Cleaning up temporary data for e26dcb2339b364f3699d110c15cd4bb3 2024-11-08T05:13:26,690 INFO [StoreOpener-12f1148a7b8db3e4f515337b8b7746fa-1 {}] regionserver.HStore(327): Store=12f1148a7b8db3e4f515337b8b7746fa/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T05:13:26,690 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1038): replaying wal for 12f1148a7b8db3e4f515337b8b7746fa 2024-11-08T05:13:26,692 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/default/test/12f1148a7b8db3e4f515337b8b7746fa 2024-11-08T05:13:26,692 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/default/test/12f1148a7b8db3e4f515337b8b7746fa 2024-11-08T05:13:26,693 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1093): writing seq id for e26dcb2339b364f3699d110c15cd4bb3 2024-11-08T05:13:26,693 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1048): stopping wal replay for 12f1148a7b8db3e4f515337b8b7746fa 2024-11-08T05:13:26,693 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1060): Cleaning up temporary data for 12f1148a7b8db3e4f515337b8b7746fa 2024-11-08T05:13:26,695 INFO [StoreOpener-64ddf686fed30cdb23b2e874efec598d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 64ddf686fed30cdb23b2e874efec598d 2024-11-08T05:13:26,697 INFO [StoreOpener-64ddf686fed30cdb23b2e874efec598d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 64ddf686fed30cdb23b2e874efec598d columnFamilyName family 2024-11-08T05:13:26,697 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1093): writing seq id for 12f1148a7b8db3e4f515337b8b7746fa 2024-11-08T05:13:26,698 DEBUG [StoreOpener-64ddf686fed30cdb23b2e874efec598d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T05:13:26,698 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/default/test/e26dcb2339b364f3699d110c15cd4bb3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T05:13:26,699 INFO [StoreOpener-64ddf686fed30cdb23b2e874efec598d-1 {}] regionserver.HStore(327): Store=64ddf686fed30cdb23b2e874efec598d/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T05:13:26,699 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1038): replaying wal for 64ddf686fed30cdb23b2e874efec598d 2024-11-08T05:13:26,700 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1114): Opened e26dcb2339b364f3699d110c15cd4bb3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72847546, jitterRate=0.08551302552223206}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T05:13:26,700 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e26dcb2339b364f3699d110c15cd4bb3 2024-11-08T05:13:26,700 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1006): Region open journal for e26dcb2339b364f3699d110c15cd4bb3: Running coprocessor pre-open hook at 1731042806659Writing region info on filesystem at 1731042806659Initializing all the Stores at 1731042806661 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731042806661Cleaning up temporary data from old regions at 1731042806689 (+28 ms)Running coprocessor post-open hooks at 1731042806700 (+11 ms)Region opened successfully at 1731042806700 2024-11-08T05:13:26,702 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/default/test/64ddf686fed30cdb23b2e874efec598d 2024-11-08T05:13:26,703 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/default/test/64ddf686fed30cdb23b2e874efec598d 2024-11-08T05:13:26,703 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=14 2024-11-08T05:13:26,703 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/default/test/12f1148a7b8db3e4f515337b8b7746fa/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T05:13:26,703 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=14, state=SUCCESS, hasLock=false; OpenRegionProcedure a12ea80f78b339cfbfc0e18b5338df0c, server=d1ba675678c9,46753,1731042802851 in 249 msec 2024-11-08T05:13:26,704 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2236): Post open deploy tasks for test,sss,1731042805591.e26dcb2339b364f3699d110c15cd4bb3., pid=48, masterSystemTime=1731042806586 2024-11-08T05:13:26,704 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1048): stopping wal replay for 64ddf686fed30cdb23b2e874efec598d 2024-11-08T05:13:26,704 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1060): Cleaning up temporary data for 64ddf686fed30cdb23b2e874efec598d 2024-11-08T05:13:26,704 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1114): Opened 12f1148a7b8db3e4f515337b8b7746fa; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69561087, jitterRate=0.03654097020626068}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T05:13:26,704 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 12f1148a7b8db3e4f515337b8b7746fa 2024-11-08T05:13:26,704 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1006): Region open journal for 12f1148a7b8db3e4f515337b8b7746fa: Running coprocessor pre-open hook at 1731042806658Writing region info on filesystem at 1731042806658Initializing all the Stores at 1731042806660 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731042806660Cleaning up temporary data from old regions at 1731042806693 (+33 ms)Running coprocessor post-open hooks at 1731042806704 (+11 ms)Region opened successfully at 1731042806704 2024-11-08T05:13:26,708 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1093): writing seq id for 64ddf686fed30cdb23b2e874efec598d 2024-11-08T05:13:26,708 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2236): Post open deploy tasks for test,rrr,1731042805591.12f1148a7b8db3e4f515337b8b7746fa., pid=53, masterSystemTime=1731042806641 2024-11-08T05:13:26,714 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2266): Finished post open deploy task for test,sss,1731042805591.e26dcb2339b364f3699d110c15cd4bb3. 2024-11-08T05:13:26,714 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(153): Opened test,sss,1731042805591.e26dcb2339b364f3699d110c15cd4bb3. 2024-11-08T05:13:26,714 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(132): Open test,ppp,1731042805591.b0ebd9a9794acc56e430b36d1f17621c. 2024-11-08T05:13:26,714 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7752): Opening region: {ENCODED => b0ebd9a9794acc56e430b36d1f17621c, NAME => 'test,ppp,1731042805591.b0ebd9a9794acc56e430b36d1f17621c.', STARTKEY => 'ppp', ENDKEY => 'qqq'} 2024-11-08T05:13:26,715 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=a12ea80f78b339cfbfc0e18b5338df0c, ASSIGN in 459 msec 2024-11-08T05:13:26,715 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test b0ebd9a9794acc56e430b36d1f17621c 2024-11-08T05:13:26,715 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(898): Instantiated test,ppp,1731042805591.b0ebd9a9794acc56e430b36d1f17621c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T05:13:26,715 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7794): checking encryption for b0ebd9a9794acc56e430b36d1f17621c 2024-11-08T05:13:26,715 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7797): checking classloading for b0ebd9a9794acc56e430b36d1f17621c 2024-11-08T05:13:26,716 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2266): Finished post open deploy task for test,rrr,1731042805591.12f1148a7b8db3e4f515337b8b7746fa. 2024-11-08T05:13:26,716 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(153): Opened test,rrr,1731042805591.12f1148a7b8db3e4f515337b8b7746fa. 2024-11-08T05:13:26,716 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=e26dcb2339b364f3699d110c15cd4bb3, regionState=OPEN, openSeqNum=2, regionLocation=d1ba675678c9,41485,1731042802970 2024-11-08T05:13:26,716 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(132): Open test,uuu,1731042805591.14efcc1b9822cc74f4d4a96648555167. 2024-11-08T05:13:26,717 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7752): Opening region: {ENCODED => 14efcc1b9822cc74f4d4a96648555167, NAME => 'test,uuu,1731042805591.14efcc1b9822cc74f4d4a96648555167.', STARTKEY => 'uuu', ENDKEY => 'vvv'} 2024-11-08T05:13:26,717 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 14efcc1b9822cc74f4d4a96648555167 2024-11-08T05:13:26,717 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(898): Instantiated test,uuu,1731042805591.14efcc1b9822cc74f4d4a96648555167.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T05:13:26,718 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7794): checking encryption for 14efcc1b9822cc74f4d4a96648555167 2024-11-08T05:13:26,718 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7797): checking classloading for 14efcc1b9822cc74f4d4a96648555167 2024-11-08T05:13:26,720 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=48, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure e26dcb2339b364f3699d110c15cd4bb3, server=d1ba675678c9,41485,1731042802970 because future has completed 2024-11-08T05:13:26,722 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=12f1148a7b8db3e4f515337b8b7746fa, regionState=OPEN, openSeqNum=2, regionLocation=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:26,724 INFO [StoreOpener-b0ebd9a9794acc56e430b36d1f17621c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region b0ebd9a9794acc56e430b36d1f17621c 2024-11-08T05:13:26,727 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=53, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure 12f1148a7b8db3e4f515337b8b7746fa, server=d1ba675678c9,39291,1731042803020 because future has completed 2024-11-08T05:13:26,728 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/default/test/64ddf686fed30cdb23b2e874efec598d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T05:13:26,729 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1114): Opened 64ddf686fed30cdb23b2e874efec598d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65075394, jitterRate=-0.030301064252853394}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T05:13:26,731 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 64ddf686fed30cdb23b2e874efec598d 2024-11-08T05:13:26,732 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37587 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=d1ba675678c9,39291,1731042803020, table=test, region=12f1148a7b8db3e4f515337b8b7746fa. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-08T05:13:26,733 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37587 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=d1ba675678c9,41485,1731042802970, table=test, region=65d5312dbd2f96bf3b16bdfd7504dff6. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-08T05:13:26,736 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1006): Region open journal for 64ddf686fed30cdb23b2e874efec598d: Running coprocessor pre-open hook at 1731042806685Writing region info on filesystem at 1731042806685Initializing all the Stores at 1731042806689 (+4 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731042806689Cleaning up temporary data from old regions at 1731042806704 (+15 ms)Running coprocessor post-open hooks at 1731042806731 (+27 ms)Region opened successfully at 1731042806736 (+5 ms) 2024-11-08T05:13:26,737 INFO [StoreOpener-b0ebd9a9794acc56e430b36d1f17621c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b0ebd9a9794acc56e430b36d1f17621c columnFamilyName family 2024-11-08T05:13:26,737 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=48, resume processing ppid=23 2024-11-08T05:13:26,737 DEBUG [StoreOpener-b0ebd9a9794acc56e430b36d1f17621c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T05:13:26,737 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, ppid=23, state=SUCCESS, hasLock=false; OpenRegionProcedure e26dcb2339b364f3699d110c15cd4bb3, server=d1ba675678c9,41485,1731042802970 in 215 msec 2024-11-08T05:13:26,737 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ddd,1731042805591.64ddf686fed30cdb23b2e874efec598d., pid=35, masterSystemTime=1731042806567 2024-11-08T05:13:26,738 INFO [StoreOpener-b0ebd9a9794acc56e430b36d1f17621c-1 {}] regionserver.HStore(327): Store=b0ebd9a9794acc56e430b36d1f17621c/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T05:13:26,738 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1038): replaying wal for b0ebd9a9794acc56e430b36d1f17621c 2024-11-08T05:13:26,740 INFO [StoreOpener-14efcc1b9822cc74f4d4a96648555167-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 14efcc1b9822cc74f4d4a96648555167 2024-11-08T05:13:26,741 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/default/test/b0ebd9a9794acc56e430b36d1f17621c 2024-11-08T05:13:26,741 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ddd,1731042805591.64ddf686fed30cdb23b2e874efec598d. 2024-11-08T05:13:26,741 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(153): Opened test,ddd,1731042805591.64ddf686fed30cdb23b2e874efec598d. 2024-11-08T05:13:26,742 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open test,ggg,1731042805591.9b3274d80dcb7deed617c19ac1de4a68. 2024-11-08T05:13:26,742 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=e26dcb2339b364f3699d110c15cd4bb3, ASSIGN in 493 msec 2024-11-08T05:13:26,742 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => 9b3274d80dcb7deed617c19ac1de4a68, NAME => 'test,ggg,1731042805591.9b3274d80dcb7deed617c19ac1de4a68.', STARTKEY => 'ggg', ENDKEY => 'hhh'} 2024-11-08T05:13:26,742 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/default/test/b0ebd9a9794acc56e430b36d1f17621c 2024-11-08T05:13:26,742 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 9b3274d80dcb7deed617c19ac1de4a68 2024-11-08T05:13:26,742 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated test,ggg,1731042805591.9b3274d80dcb7deed617c19ac1de4a68.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T05:13:26,743 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for 9b3274d80dcb7deed617c19ac1de4a68 2024-11-08T05:13:26,743 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for 9b3274d80dcb7deed617c19ac1de4a68 2024-11-08T05:13:26,743 INFO [StoreOpener-14efcc1b9822cc74f4d4a96648555167-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 14efcc1b9822cc74f4d4a96648555167 columnFamilyName family 2024-11-08T05:13:26,743 DEBUG [StoreOpener-14efcc1b9822cc74f4d4a96648555167-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T05:13:26,744 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1048): stopping wal replay for b0ebd9a9794acc56e430b36d1f17621c 2024-11-08T05:13:26,744 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1060): Cleaning up temporary data for b0ebd9a9794acc56e430b36d1f17621c 2024-11-08T05:13:26,744 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=64ddf686fed30cdb23b2e874efec598d, regionState=OPEN, openSeqNum=2, regionLocation=d1ba675678c9,46753,1731042802851 2024-11-08T05:13:26,746 INFO [StoreOpener-14efcc1b9822cc74f4d4a96648555167-1 {}] regionserver.HStore(327): Store=14efcc1b9822cc74f4d4a96648555167/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T05:13:26,747 INFO [StoreOpener-9b3274d80dcb7deed617c19ac1de4a68-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 9b3274d80dcb7deed617c19ac1de4a68 2024-11-08T05:13:26,749 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1038): replaying wal for 14efcc1b9822cc74f4d4a96648555167 2024-11-08T05:13:26,750 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/default/test/14efcc1b9822cc74f4d4a96648555167 2024-11-08T05:13:26,751 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/default/test/14efcc1b9822cc74f4d4a96648555167 2024-11-08T05:13:26,752 INFO [StoreOpener-9b3274d80dcb7deed617c19ac1de4a68-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9b3274d80dcb7deed617c19ac1de4a68 columnFamilyName family 2024-11-08T05:13:26,752 DEBUG [StoreOpener-9b3274d80dcb7deed617c19ac1de4a68-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T05:13:26,752 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1093): writing seq id for b0ebd9a9794acc56e430b36d1f17621c 2024-11-08T05:13:26,752 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 64ddf686fed30cdb23b2e874efec598d, server=d1ba675678c9,46753,1731042802851 because future has completed 2024-11-08T05:13:26,753 INFO [StoreOpener-9b3274d80dcb7deed617c19ac1de4a68-1 {}] regionserver.HStore(327): Store=9b3274d80dcb7deed617c19ac1de4a68/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T05:13:26,753 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1048): stopping wal replay for 14efcc1b9822cc74f4d4a96648555167 2024-11-08T05:13:26,753 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1060): Cleaning up temporary data for 14efcc1b9822cc74f4d4a96648555167 2024-11-08T05:13:26,753 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for 9b3274d80dcb7deed617c19ac1de4a68 2024-11-08T05:13:26,756 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/default/test/9b3274d80dcb7deed617c19ac1de4a68 2024-11-08T05:13:26,757 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/default/test/9b3274d80dcb7deed617c19ac1de4a68 2024-11-08T05:13:26,757 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/default/test/b0ebd9a9794acc56e430b36d1f17621c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T05:13:26,758 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for 9b3274d80dcb7deed617c19ac1de4a68 2024-11-08T05:13:26,758 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for 9b3274d80dcb7deed617c19ac1de4a68 2024-11-08T05:13:26,758 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1093): writing seq id for 14efcc1b9822cc74f4d4a96648555167 2024-11-08T05:13:26,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37587 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-08T05:13:26,760 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1114): Opened b0ebd9a9794acc56e430b36d1f17621c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67543265, jitterRate=0.006473079323768616}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T05:13:26,760 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b0ebd9a9794acc56e430b36d1f17621c 2024-11-08T05:13:26,760 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1006): Region open journal for b0ebd9a9794acc56e430b36d1f17621c: Running coprocessor pre-open hook at 1731042806716Writing region info on filesystem at 1731042806716Initializing all the Stores at 1731042806717 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731042806717Cleaning up temporary data from old regions at 1731042806744 (+27 ms)Running coprocessor post-open hooks at 1731042806760 (+16 ms)Region opened successfully at 1731042806760 2024-11-08T05:13:26,762 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ppp,1731042805591.b0ebd9a9794acc56e430b36d1f17621c., pid=47, masterSystemTime=1731042806586 2024-11-08T05:13:26,763 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for 9b3274d80dcb7deed617c19ac1de4a68 2024-11-08T05:13:26,763 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/default/test/14efcc1b9822cc74f4d4a96648555167/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T05:13:26,764 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=8 2024-11-08T05:13:26,764 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 64ddf686fed30cdb23b2e874efec598d, server=d1ba675678c9,46753,1731042802851 in 319 msec 2024-11-08T05:13:26,765 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1114): Opened 14efcc1b9822cc74f4d4a96648555167; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69381533, jitterRate=0.03386540710926056}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T05:13:26,766 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 14efcc1b9822cc74f4d4a96648555167 2024-11-08T05:13:26,766 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1006): Region open journal for 14efcc1b9822cc74f4d4a96648555167: Running coprocessor pre-open hook at 1731042806718Writing region info on filesystem at 1731042806718Initializing all the Stores at 1731042806721 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731042806722 (+1 ms)Cleaning up temporary data from old regions at 1731042806753 (+31 ms)Running coprocessor post-open hooks at 1731042806766 (+13 ms)Region opened successfully at 1731042806766 2024-11-08T05:13:26,766 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ppp,1731042805591.b0ebd9a9794acc56e430b36d1f17621c. 2024-11-08T05:13:26,767 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(153): Opened test,ppp,1731042805591.b0ebd9a9794acc56e430b36d1f17621c. 2024-11-08T05:13:26,767 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(132): Open test,yyy,1731042805591.de358f808110a48fa32e0533fefdc95c. 2024-11-08T05:13:26,768 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2236): Post open deploy tasks for test,uuu,1731042805591.14efcc1b9822cc74f4d4a96648555167., pid=49, masterSystemTime=1731042806641 2024-11-08T05:13:26,768 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7752): Opening region: {ENCODED => de358f808110a48fa32e0533fefdc95c, NAME => 'test,yyy,1731042805591.de358f808110a48fa32e0533fefdc95c.', STARTKEY => 'yyy', ENDKEY => 'zzz'} 2024-11-08T05:13:26,768 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test de358f808110a48fa32e0533fefdc95c 2024-11-08T05:13:26,768 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(898): Instantiated test,yyy,1731042805591.de358f808110a48fa32e0533fefdc95c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T05:13:26,768 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7794): checking encryption for de358f808110a48fa32e0533fefdc95c 2024-11-08T05:13:26,768 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7797): checking classloading for de358f808110a48fa32e0533fefdc95c 2024-11-08T05:13:26,768 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=64ddf686fed30cdb23b2e874efec598d, ASSIGN in 520 msec 2024-11-08T05:13:26,771 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=b0ebd9a9794acc56e430b36d1f17621c, regionState=OPEN, openSeqNum=2, regionLocation=d1ba675678c9,41485,1731042802970 2024-11-08T05:13:26,772 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2266): Finished post open deploy task for test,uuu,1731042805591.14efcc1b9822cc74f4d4a96648555167. 2024-11-08T05:13:26,772 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(153): Opened test,uuu,1731042805591.14efcc1b9822cc74f4d4a96648555167. 2024-11-08T05:13:26,772 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(132): Open test,,1731042805591.6240e38715889f3bb77bb89478c81a6c. 2024-11-08T05:13:26,772 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7752): Opening region: {ENCODED => 6240e38715889f3bb77bb89478c81a6c, NAME => 'test,,1731042805591.6240e38715889f3bb77bb89478c81a6c.', STARTKEY => '', ENDKEY => 'bbb'} 2024-11-08T05:13:26,773 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 6240e38715889f3bb77bb89478c81a6c 2024-11-08T05:13:26,773 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(898): Instantiated test,,1731042805591.6240e38715889f3bb77bb89478c81a6c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T05:13:26,773 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/default/test/9b3274d80dcb7deed617c19ac1de4a68/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T05:13:26,774 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened 9b3274d80dcb7deed617c19ac1de4a68; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73435203, jitterRate=0.09426979720592499}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T05:13:26,775 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9b3274d80dcb7deed617c19ac1de4a68 2024-11-08T05:13:26,775 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7794): checking encryption for 6240e38715889f3bb77bb89478c81a6c 2024-11-08T05:13:26,775 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=47, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure b0ebd9a9794acc56e430b36d1f17621c, server=d1ba675678c9,41485,1731042802970 because future has completed 2024-11-08T05:13:26,775 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for 9b3274d80dcb7deed617c19ac1de4a68: Running coprocessor pre-open hook at 1731042806743Writing region info on filesystem at 1731042806743Initializing all the Stores at 1731042806745 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731042806745Cleaning up temporary data from old regions at 1731042806758 (+13 ms)Running coprocessor post-open hooks at 1731042806775 (+17 ms)Region opened successfully at 1731042806775 2024-11-08T05:13:26,775 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7797): checking classloading for 6240e38715889f3bb77bb89478c81a6c 2024-11-08T05:13:26,776 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=14efcc1b9822cc74f4d4a96648555167, regionState=OPEN, openSeqNum=2, regionLocation=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:26,777 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ggg,1731042805591.9b3274d80dcb7deed617c19ac1de4a68., pid=33, masterSystemTime=1731042806567 2024-11-08T05:13:26,781 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ggg,1731042805591.9b3274d80dcb7deed617c19ac1de4a68. 2024-11-08T05:13:26,781 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened test,ggg,1731042805591.9b3274d80dcb7deed617c19ac1de4a68. 2024-11-08T05:13:26,781 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(132): Open test,ttt,1731042805591.445b14856951586918c91d4ba342f1ff. 2024-11-08T05:13:26,781 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7752): Opening region: {ENCODED => 445b14856951586918c91d4ba342f1ff, NAME => 'test,ttt,1731042805591.445b14856951586918c91d4ba342f1ff.', STARTKEY => 'ttt', ENDKEY => 'uuu'} 2024-11-08T05:13:26,782 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 445b14856951586918c91d4ba342f1ff 2024-11-08T05:13:26,782 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(898): Instantiated test,ttt,1731042805591.445b14856951586918c91d4ba342f1ff.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T05:13:26,782 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7794): checking encryption for 445b14856951586918c91d4ba342f1ff 2024-11-08T05:13:26,782 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7797): checking classloading for 445b14856951586918c91d4ba342f1ff 2024-11-08T05:13:26,783 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=49, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure 14efcc1b9822cc74f4d4a96648555167, server=d1ba675678c9,39291,1731042803020 because future has completed 2024-11-08T05:13:26,784 INFO [StoreOpener-de358f808110a48fa32e0533fefdc95c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region de358f808110a48fa32e0533fefdc95c 2024-11-08T05:13:26,784 INFO [StoreOpener-6240e38715889f3bb77bb89478c81a6c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 6240e38715889f3bb77bb89478c81a6c 2024-11-08T05:13:26,785 INFO [StoreOpener-445b14856951586918c91d4ba342f1ff-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 445b14856951586918c91d4ba342f1ff 2024-11-08T05:13:26,786 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=20 2024-11-08T05:13:26,787 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure b0ebd9a9794acc56e430b36d1f17621c, server=d1ba675678c9,41485,1731042802970 in 273 msec 2024-11-08T05:13:26,787 INFO [StoreOpener-6240e38715889f3bb77bb89478c81a6c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6240e38715889f3bb77bb89478c81a6c columnFamilyName family 2024-11-08T05:13:26,787 DEBUG [StoreOpener-6240e38715889f3bb77bb89478c81a6c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T05:13:26,787 INFO [StoreOpener-445b14856951586918c91d4ba342f1ff-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 445b14856951586918c91d4ba342f1ff columnFamilyName family 2024-11-08T05:13:26,787 INFO [StoreOpener-de358f808110a48fa32e0533fefdc95c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region de358f808110a48fa32e0533fefdc95c columnFamilyName family 2024-11-08T05:13:26,787 DEBUG [StoreOpener-445b14856951586918c91d4ba342f1ff-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T05:13:26,787 DEBUG [StoreOpener-de358f808110a48fa32e0533fefdc95c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T05:13:26,788 INFO [StoreOpener-6240e38715889f3bb77bb89478c81a6c-1 {}] regionserver.HStore(327): Store=6240e38715889f3bb77bb89478c81a6c/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T05:13:26,788 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1038): replaying wal for 6240e38715889f3bb77bb89478c81a6c 2024-11-08T05:13:26,788 INFO [StoreOpener-de358f808110a48fa32e0533fefdc95c-1 {}] regionserver.HStore(327): Store=de358f808110a48fa32e0533fefdc95c/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T05:13:26,788 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1038): replaying wal for de358f808110a48fa32e0533fefdc95c 2024-11-08T05:13:26,788 INFO [StoreOpener-445b14856951586918c91d4ba342f1ff-1 {}] regionserver.HStore(327): Store=445b14856951586918c91d4ba342f1ff/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T05:13:26,789 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1038): replaying wal for 445b14856951586918c91d4ba342f1ff 2024-11-08T05:13:26,789 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/default/test/6240e38715889f3bb77bb89478c81a6c 2024-11-08T05:13:26,790 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/default/test/de358f808110a48fa32e0533fefdc95c 2024-11-08T05:13:26,790 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/default/test/6240e38715889f3bb77bb89478c81a6c 2024-11-08T05:13:26,790 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/default/test/445b14856951586918c91d4ba342f1ff 2024-11-08T05:13:26,790 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=b0ebd9a9794acc56e430b36d1f17621c, ASSIGN in 543 msec 2024-11-08T05:13:26,790 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/default/test/de358f808110a48fa32e0533fefdc95c 2024-11-08T05:13:26,791 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1048): stopping wal replay for 6240e38715889f3bb77bb89478c81a6c 2024-11-08T05:13:26,791 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1060): Cleaning up temporary data for 6240e38715889f3bb77bb89478c81a6c 2024-11-08T05:13:26,791 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1048): stopping wal replay for de358f808110a48fa32e0533fefdc95c 2024-11-08T05:13:26,792 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1060): Cleaning up temporary data for de358f808110a48fa32e0533fefdc95c 2024-11-08T05:13:26,792 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/default/test/445b14856951586918c91d4ba342f1ff 2024-11-08T05:13:26,793 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1048): stopping wal replay for 445b14856951586918c91d4ba342f1ff 2024-11-08T05:13:26,793 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1060): Cleaning up temporary data for 445b14856951586918c91d4ba342f1ff 2024-11-08T05:13:26,796 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1093): writing seq id for 6240e38715889f3bb77bb89478c81a6c 2024-11-08T05:13:26,796 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1093): writing seq id for 445b14856951586918c91d4ba342f1ff 2024-11-08T05:13:26,796 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1093): writing seq id for de358f808110a48fa32e0533fefdc95c 2024-11-08T05:13:26,796 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=25 2024-11-08T05:13:26,797 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=25, state=SUCCESS, hasLock=false; OpenRegionProcedure 14efcc1b9822cc74f4d4a96648555167, server=d1ba675678c9,39291,1731042803020 in 269 msec 2024-11-08T05:13:26,800 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/default/test/de358f808110a48fa32e0533fefdc95c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T05:13:26,800 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=14efcc1b9822cc74f4d4a96648555167, ASSIGN in 553 msec 2024-11-08T05:13:26,801 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1114): Opened de358f808110a48fa32e0533fefdc95c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69337310, jitterRate=0.033206433057785034}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T05:13:26,801 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/default/test/6240e38715889f3bb77bb89478c81a6c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T05:13:26,801 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1122): Running coprocessor post-open hooks for de358f808110a48fa32e0533fefdc95c 2024-11-08T05:13:26,801 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1006): Region open journal for de358f808110a48fa32e0533fefdc95c: Running coprocessor pre-open hook at 1731042806769Writing region info on filesystem at 1731042806769Initializing all the Stores at 1731042806771 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731042806771Cleaning up temporary data from old regions at 1731042806792 (+21 ms)Running coprocessor post-open hooks at 1731042806801 (+9 ms)Region opened successfully at 1731042806801 2024-11-08T05:13:26,802 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1114): Opened 6240e38715889f3bb77bb89478c81a6c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64259820, jitterRate=-0.0424540638923645}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T05:13:26,802 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6240e38715889f3bb77bb89478c81a6c 2024-11-08T05:13:26,802 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1006): Region open journal for 6240e38715889f3bb77bb89478c81a6c: Running coprocessor pre-open hook at 1731042806775Writing region info on filesystem at 1731042806775Initializing all the Stores at 1731042806778 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731042806778Cleaning up temporary data from old regions at 1731042806791 (+13 ms)Running coprocessor post-open hooks at 1731042806802 (+11 ms)Region opened successfully at 1731042806802 2024-11-08T05:13:26,803 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=9b3274d80dcb7deed617c19ac1de4a68, regionState=OPEN, openSeqNum=2, regionLocation=d1ba675678c9,46753,1731042802851 2024-11-08T05:13:26,803 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2236): Post open deploy tasks for test,yyy,1731042805591.de358f808110a48fa32e0533fefdc95c., pid=34, masterSystemTime=1731042806586 2024-11-08T05:13:26,803 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2236): Post open deploy tasks for test,,1731042805591.6240e38715889f3bb77bb89478c81a6c., pid=55, masterSystemTime=1731042806641 2024-11-08T05:13:26,806 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9b3274d80dcb7deed617c19ac1de4a68, server=d1ba675678c9,46753,1731042802851 because future has completed 2024-11-08T05:13:26,806 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/default/test/445b14856951586918c91d4ba342f1ff/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T05:13:26,807 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2266): Finished post open deploy task for test,,1731042805591.6240e38715889f3bb77bb89478c81a6c. 2024-11-08T05:13:26,807 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(153): Opened test,,1731042805591.6240e38715889f3bb77bb89478c81a6c. 2024-11-08T05:13:26,807 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(132): Open test,xxx,1731042805591.87b3455cffd1f4e06e5901b3bcbed9dc. 2024-11-08T05:13:26,807 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7752): Opening region: {ENCODED => 87b3455cffd1f4e06e5901b3bcbed9dc, NAME => 'test,xxx,1731042805591.87b3455cffd1f4e06e5901b3bcbed9dc.', STARTKEY => 'xxx', ENDKEY => 'yyy'} 2024-11-08T05:13:26,808 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 87b3455cffd1f4e06e5901b3bcbed9dc 2024-11-08T05:13:26,808 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(898): Instantiated test,xxx,1731042805591.87b3455cffd1f4e06e5901b3bcbed9dc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T05:13:26,808 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7794): checking encryption for 87b3455cffd1f4e06e5901b3bcbed9dc 2024-11-08T05:13:26,808 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=6240e38715889f3bb77bb89478c81a6c, regionState=OPEN, openSeqNum=2, regionLocation=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:26,808 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7797): checking classloading for 87b3455cffd1f4e06e5901b3bcbed9dc 2024-11-08T05:13:26,808 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2266): Finished post open deploy task for test,yyy,1731042805591.de358f808110a48fa32e0533fefdc95c. 2024-11-08T05:13:26,809 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(153): Opened test,yyy,1731042805591.de358f808110a48fa32e0533fefdc95c. 2024-11-08T05:13:26,809 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(132): Open test,vvv,1731042805591.7c8f72849b2e1719ed903afc73e0571a. 2024-11-08T05:13:26,809 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7752): Opening region: {ENCODED => 7c8f72849b2e1719ed903afc73e0571a, NAME => 'test,vvv,1731042805591.7c8f72849b2e1719ed903afc73e0571a.', STARTKEY => 'vvv', ENDKEY => 'www'} 2024-11-08T05:13:26,809 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 7c8f72849b2e1719ed903afc73e0571a 2024-11-08T05:13:26,809 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(898): Instantiated test,vvv,1731042805591.7c8f72849b2e1719ed903afc73e0571a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T05:13:26,810 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7794): checking encryption for 7c8f72849b2e1719ed903afc73e0571a 2024-11-08T05:13:26,810 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7797): checking classloading for 7c8f72849b2e1719ed903afc73e0571a 2024-11-08T05:13:26,810 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1114): Opened 445b14856951586918c91d4ba342f1ff; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62664788, jitterRate=-0.06622189283370972}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T05:13:26,810 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 445b14856951586918c91d4ba342f1ff 2024-11-08T05:13:26,810 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1006): Region open journal for 445b14856951586918c91d4ba342f1ff: Running coprocessor pre-open hook at 1731042806782Writing region info on filesystem at 1731042806782Initializing all the Stores at 1731042806784 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731042806784Cleaning up temporary data from old regions at 1731042806793 (+9 ms)Running coprocessor post-open hooks at 1731042806810 (+17 ms)Region opened successfully at 1731042806810 2024-11-08T05:13:26,812 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ttt,1731042805591.445b14856951586918c91d4ba342f1ff., pid=38, masterSystemTime=1731042806567 2024-11-08T05:13:26,812 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=55, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6240e38715889f3bb77bb89478c81a6c, server=d1ba675678c9,39291,1731042803020 because future has completed 2024-11-08T05:13:26,813 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=de358f808110a48fa32e0533fefdc95c, regionState=OPEN, openSeqNum=2, regionLocation=d1ba675678c9,41485,1731042802970 2024-11-08T05:13:26,816 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure de358f808110a48fa32e0533fefdc95c, server=d1ba675678c9,41485,1731042802970 because future has completed 2024-11-08T05:13:26,816 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ttt,1731042805591.445b14856951586918c91d4ba342f1ff. 2024-11-08T05:13:26,816 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(153): Opened test,ttt,1731042805591.445b14856951586918c91d4ba342f1ff. 2024-11-08T05:13:26,816 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(132): Open test,www,1731042805591.81806cb184c70c9dc0df408694092d85. 2024-11-08T05:13:26,816 INFO [StoreOpener-87b3455cffd1f4e06e5901b3bcbed9dc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 87b3455cffd1f4e06e5901b3bcbed9dc 2024-11-08T05:13:26,816 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7752): Opening region: {ENCODED => 81806cb184c70c9dc0df408694092d85, NAME => 'test,www,1731042805591.81806cb184c70c9dc0df408694092d85.', STARTKEY => 'www', ENDKEY => 'xxx'} 2024-11-08T05:13:26,817 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 81806cb184c70c9dc0df408694092d85 2024-11-08T05:13:26,817 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(898): Instantiated test,www,1731042805591.81806cb184c70c9dc0df408694092d85.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T05:13:26,817 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7794): checking encryption for 81806cb184c70c9dc0df408694092d85 2024-11-08T05:13:26,817 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7797): checking classloading for 81806cb184c70c9dc0df408694092d85 2024-11-08T05:13:26,817 INFO [StoreOpener-7c8f72849b2e1719ed903afc73e0571a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 7c8f72849b2e1719ed903afc73e0571a 2024-11-08T05:13:26,819 INFO [StoreOpener-87b3455cffd1f4e06e5901b3bcbed9dc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 87b3455cffd1f4e06e5901b3bcbed9dc columnFamilyName family 2024-11-08T05:13:26,819 DEBUG [StoreOpener-87b3455cffd1f4e06e5901b3bcbed9dc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T05:13:26,820 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=445b14856951586918c91d4ba342f1ff, regionState=OPEN, openSeqNum=2, regionLocation=d1ba675678c9,46753,1731042802851 2024-11-08T05:13:26,820 INFO [StoreOpener-87b3455cffd1f4e06e5901b3bcbed9dc-1 {}] regionserver.HStore(327): Store=87b3455cffd1f4e06e5901b3bcbed9dc/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T05:13:26,820 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1038): replaying wal for 87b3455cffd1f4e06e5901b3bcbed9dc 2024-11-08T05:13:26,821 INFO [StoreOpener-7c8f72849b2e1719ed903afc73e0571a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7c8f72849b2e1719ed903afc73e0571a columnFamilyName family 2024-11-08T05:13:26,821 DEBUG [StoreOpener-7c8f72849b2e1719ed903afc73e0571a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T05:13:26,822 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/default/test/87b3455cffd1f4e06e5901b3bcbed9dc 2024-11-08T05:13:26,822 INFO [StoreOpener-7c8f72849b2e1719ed903afc73e0571a-1 {}] regionserver.HStore(327): Store=7c8f72849b2e1719ed903afc73e0571a/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T05:13:26,822 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1038): replaying wal for 7c8f72849b2e1719ed903afc73e0571a 2024-11-08T05:13:26,823 INFO [StoreOpener-81806cb184c70c9dc0df408694092d85-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 81806cb184c70c9dc0df408694092d85 2024-11-08T05:13:26,823 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/default/test/87b3455cffd1f4e06e5901b3bcbed9dc 2024-11-08T05:13:26,824 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure 445b14856951586918c91d4ba342f1ff, server=d1ba675678c9,46753,1731042802851 because future has completed 2024-11-08T05:13:26,824 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/default/test/7c8f72849b2e1719ed903afc73e0571a 2024-11-08T05:13:26,824 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1048): stopping wal replay for 87b3455cffd1f4e06e5901b3bcbed9dc 2024-11-08T05:13:26,825 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1060): Cleaning up temporary data for 87b3455cffd1f4e06e5901b3bcbed9dc 2024-11-08T05:13:26,825 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/default/test/7c8f72849b2e1719ed903afc73e0571a 2024-11-08T05:13:26,826 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1048): stopping wal replay for 7c8f72849b2e1719ed903afc73e0571a 2024-11-08T05:13:26,826 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1060): Cleaning up temporary data for 7c8f72849b2e1719ed903afc73e0571a 2024-11-08T05:13:26,826 INFO [StoreOpener-81806cb184c70c9dc0df408694092d85-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 81806cb184c70c9dc0df408694092d85 columnFamilyName family 2024-11-08T05:13:26,826 DEBUG [StoreOpener-81806cb184c70c9dc0df408694092d85-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T05:13:26,827 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1093): writing seq id for 87b3455cffd1f4e06e5901b3bcbed9dc 2024-11-08T05:13:26,828 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1093): writing seq id for 7c8f72849b2e1719ed903afc73e0571a 2024-11-08T05:13:26,830 INFO [StoreOpener-81806cb184c70c9dc0df408694092d85-1 {}] regionserver.HStore(327): Store=81806cb184c70c9dc0df408694092d85/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T05:13:26,830 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1038): replaying wal for 81806cb184c70c9dc0df408694092d85 2024-11-08T05:13:26,831 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/default/test/81806cb184c70c9dc0df408694092d85 2024-11-08T05:13:26,831 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/default/test/87b3455cffd1f4e06e5901b3bcbed9dc/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T05:13:26,832 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1114): Opened 87b3455cffd1f4e06e5901b3bcbed9dc; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59991108, jitterRate=-0.10606282949447632}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T05:13:26,832 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/default/test/7c8f72849b2e1719ed903afc73e0571a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T05:13:26,832 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/default/test/81806cb184c70c9dc0df408694092d85 2024-11-08T05:13:26,832 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 87b3455cffd1f4e06e5901b3bcbed9dc 2024-11-08T05:13:26,832 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1006): Region open journal for 87b3455cffd1f4e06e5901b3bcbed9dc: Running coprocessor pre-open hook at 1731042806808Writing region info on filesystem at 1731042806808Initializing all the Stores at 1731042806812 (+4 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731042806812Cleaning up temporary data from old regions at 1731042806825 (+13 ms)Running coprocessor post-open hooks at 1731042806832 (+7 ms)Region opened successfully at 1731042806832 2024-11-08T05:13:26,834 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1048): stopping wal replay for 81806cb184c70c9dc0df408694092d85 2024-11-08T05:13:26,834 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=29 2024-11-08T05:13:26,834 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1060): Cleaning up temporary data for 81806cb184c70c9dc0df408694092d85 2024-11-08T05:13:26,834 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure de358f808110a48fa32e0533fefdc95c, server=d1ba675678c9,41485,1731042802970 in 393 msec 2024-11-08T05:13:26,834 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1114): Opened 7c8f72849b2e1719ed903afc73e0571a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68530296, jitterRate=0.02118098735809326}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T05:13:26,834 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7c8f72849b2e1719ed903afc73e0571a 2024-11-08T05:13:26,835 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1006): Region open journal for 7c8f72849b2e1719ed903afc73e0571a: Running coprocessor pre-open hook at 1731042806810Writing region info on filesystem at 1731042806810Initializing all the Stores at 1731042806813 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731042806813Cleaning up temporary data from old regions at 1731042806826 (+13 ms)Running coprocessor post-open hooks at 1731042806834 (+8 ms)Region opened successfully at 1731042806834 2024-11-08T05:13:26,836 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37587 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=d1ba675678c9,39291,1731042803020, table=test, region=6240e38715889f3bb77bb89478c81a6c. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-08T05:13:26,836 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37587 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=d1ba675678c9,39291,1731042803020, table=test, region=12f1148a7b8db3e4f515337b8b7746fa. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-08T05:13:26,836 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2236): Post open deploy tasks for test,xxx,1731042805591.87b3455cffd1f4e06e5901b3bcbed9dc., pid=43, masterSystemTime=1731042806641 2024-11-08T05:13:26,837 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2236): Post open deploy tasks for test,vvv,1731042805591.7c8f72849b2e1719ed903afc73e0571a., pid=40, masterSystemTime=1731042806586 2024-11-08T05:13:26,838 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37587 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=d1ba675678c9,41485,1731042802970, table=test, region=65d5312dbd2f96bf3b16bdfd7504dff6. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-08T05:13:26,840 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=de358f808110a48fa32e0533fefdc95c, ASSIGN in 591 msec 2024-11-08T05:13:26,840 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1093): writing seq id for 81806cb184c70c9dc0df408694092d85 2024-11-08T05:13:26,840 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2266): Finished post open deploy task for test,xxx,1731042805591.87b3455cffd1f4e06e5901b3bcbed9dc. 2024-11-08T05:13:26,840 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(153): Opened test,xxx,1731042805591.87b3455cffd1f4e06e5901b3bcbed9dc. 2024-11-08T05:13:26,840 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(132): Open test,hhh,1731042805591.2d23e3e11ecc480e314e989d1bb8d0c3. 2024-11-08T05:13:26,841 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7752): Opening region: {ENCODED => 2d23e3e11ecc480e314e989d1bb8d0c3, NAME => 'test,hhh,1731042805591.2d23e3e11ecc480e314e989d1bb8d0c3.', STARTKEY => 'hhh', ENDKEY => 'iii'} 2024-11-08T05:13:26,841 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 2d23e3e11ecc480e314e989d1bb8d0c3 2024-11-08T05:13:26,841 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(898): Instantiated test,hhh,1731042805591.2d23e3e11ecc480e314e989d1bb8d0c3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T05:13:26,841 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7794): checking encryption for 2d23e3e11ecc480e314e989d1bb8d0c3 2024-11-08T05:13:26,841 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7797): checking classloading for 2d23e3e11ecc480e314e989d1bb8d0c3 2024-11-08T05:13:26,842 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2266): Finished post open deploy task for test,vvv,1731042805591.7c8f72849b2e1719ed903afc73e0571a. 2024-11-08T05:13:26,842 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(153): Opened test,vvv,1731042805591.7c8f72849b2e1719ed903afc73e0571a. 2024-11-08T05:13:26,842 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(132): Open test,mmm,1731042805591.27c9aaf34aaf98cd5bc8030630ae7fe3. 2024-11-08T05:13:26,843 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7752): Opening region: {ENCODED => 27c9aaf34aaf98cd5bc8030630ae7fe3, NAME => 'test,mmm,1731042805591.27c9aaf34aaf98cd5bc8030630ae7fe3.', STARTKEY => 'mmm', ENDKEY => 'nnn'} 2024-11-08T05:13:26,843 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 27c9aaf34aaf98cd5bc8030630ae7fe3 2024-11-08T05:13:26,843 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(898): Instantiated test,mmm,1731042805591.27c9aaf34aaf98cd5bc8030630ae7fe3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T05:13:26,843 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7794): checking encryption for 27c9aaf34aaf98cd5bc8030630ae7fe3 2024-11-08T05:13:26,843 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7797): checking classloading for 27c9aaf34aaf98cd5bc8030630ae7fe3 2024-11-08T05:13:26,844 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=87b3455cffd1f4e06e5901b3bcbed9dc, regionState=OPEN, openSeqNum=2, regionLocation=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:26,845 INFO [StoreOpener-2d23e3e11ecc480e314e989d1bb8d0c3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 2d23e3e11ecc480e314e989d1bb8d0c3 2024-11-08T05:13:26,846 INFO [StoreOpener-27c9aaf34aaf98cd5bc8030630ae7fe3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 27c9aaf34aaf98cd5bc8030630ae7fe3 2024-11-08T05:13:26,848 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=43, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure 87b3455cffd1f4e06e5901b3bcbed9dc, server=d1ba675678c9,39291,1731042803020 because future has completed 2024-11-08T05:13:26,848 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/default/test/81806cb184c70c9dc0df408694092d85/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T05:13:26,848 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=7c8f72849b2e1719ed903afc73e0571a, regionState=OPEN, openSeqNum=2, regionLocation=d1ba675678c9,41485,1731042802970 2024-11-08T05:13:26,849 INFO [StoreOpener-2d23e3e11ecc480e314e989d1bb8d0c3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2d23e3e11ecc480e314e989d1bb8d0c3 columnFamilyName family 2024-11-08T05:13:26,849 DEBUG [StoreOpener-2d23e3e11ecc480e314e989d1bb8d0c3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T05:13:26,849 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1114): Opened 81806cb184c70c9dc0df408694092d85; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74380196, jitterRate=0.10835129022598267}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T05:13:26,849 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 81806cb184c70c9dc0df408694092d85 2024-11-08T05:13:26,849 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1006): Region open journal for 81806cb184c70c9dc0df408694092d85: Running coprocessor pre-open hook at 1731042806817Writing region info on filesystem at 1731042806817Initializing all the Stores at 1731042806819 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731042806819Cleaning up temporary data from old regions at 1731042806834 (+15 ms)Running coprocessor post-open hooks at 1731042806849 (+15 ms)Region opened successfully at 1731042806849 2024-11-08T05:13:26,850 INFO [StoreOpener-2d23e3e11ecc480e314e989d1bb8d0c3-1 {}] regionserver.HStore(327): Store=2d23e3e11ecc480e314e989d1bb8d0c3/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T05:13:26,850 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1038): replaying wal for 2d23e3e11ecc480e314e989d1bb8d0c3 2024-11-08T05:13:26,851 INFO [StoreOpener-27c9aaf34aaf98cd5bc8030630ae7fe3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 27c9aaf34aaf98cd5bc8030630ae7fe3 columnFamilyName family 2024-11-08T05:13:26,851 DEBUG [StoreOpener-27c9aaf34aaf98cd5bc8030630ae7fe3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T05:13:26,851 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2236): Post open deploy tasks for test,www,1731042805591.81806cb184c70c9dc0df408694092d85., pid=39, masterSystemTime=1731042806567 2024-11-08T05:13:26,853 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/default/test/2d23e3e11ecc480e314e989d1bb8d0c3 2024-11-08T05:13:26,853 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=40, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7c8f72849b2e1719ed903afc73e0571a, server=d1ba675678c9,41485,1731042802970 because future has completed 2024-11-08T05:13:26,853 INFO [StoreOpener-27c9aaf34aaf98cd5bc8030630ae7fe3-1 {}] regionserver.HStore(327): Store=27c9aaf34aaf98cd5bc8030630ae7fe3/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T05:13:26,853 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1038): replaying wal for 27c9aaf34aaf98cd5bc8030630ae7fe3 2024-11-08T05:13:26,854 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/default/test/2d23e3e11ecc480e314e989d1bb8d0c3 2024-11-08T05:13:26,854 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1048): stopping wal replay for 2d23e3e11ecc480e314e989d1bb8d0c3 2024-11-08T05:13:26,854 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1060): Cleaning up temporary data for 2d23e3e11ecc480e314e989d1bb8d0c3 2024-11-08T05:13:26,855 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/default/test/27c9aaf34aaf98cd5bc8030630ae7fe3 2024-11-08T05:13:26,855 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/default/test/27c9aaf34aaf98cd5bc8030630ae7fe3 2024-11-08T05:13:26,855 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2266): Finished post open deploy task for test,www,1731042805591.81806cb184c70c9dc0df408694092d85. 2024-11-08T05:13:26,855 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(153): Opened test,www,1731042805591.81806cb184c70c9dc0df408694092d85. 2024-11-08T05:13:26,856 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1048): stopping wal replay for 27c9aaf34aaf98cd5bc8030630ae7fe3 2024-11-08T05:13:26,856 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1060): Cleaning up temporary data for 27c9aaf34aaf98cd5bc8030630ae7fe3 2024-11-08T05:13:26,857 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1093): writing seq id for 2d23e3e11ecc480e314e989d1bb8d0c3 2024-11-08T05:13:26,858 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=28 2024-11-08T05:13:26,859 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=28, state=SUCCESS, hasLock=false; OpenRegionProcedure 87b3455cffd1f4e06e5901b3bcbed9dc, server=d1ba675678c9,39291,1731042803020 in 365 msec 2024-11-08T05:13:26,859 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1093): writing seq id for 27c9aaf34aaf98cd5bc8030630ae7fe3 2024-11-08T05:13:26,861 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/default/test/2d23e3e11ecc480e314e989d1bb8d0c3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T05:13:26,862 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1114): Opened 2d23e3e11ecc480e314e989d1bb8d0c3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68184916, jitterRate=0.016034424304962158}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T05:13:26,863 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2d23e3e11ecc480e314e989d1bb8d0c3 2024-11-08T05:13:26,863 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/default/test/27c9aaf34aaf98cd5bc8030630ae7fe3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T05:13:26,863 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1006): Region open journal for 2d23e3e11ecc480e314e989d1bb8d0c3: Running coprocessor pre-open hook at 1731042806842Writing region info on filesystem at 1731042806842Initializing all the Stores at 1731042806844 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731042806844Cleaning up temporary data from old regions at 1731042806854 (+10 ms)Running coprocessor post-open hooks at 1731042806863 (+9 ms)Region opened successfully at 1731042806863 2024-11-08T05:13:26,864 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2236): Post open deploy tasks for test,hhh,1731042805591.2d23e3e11ecc480e314e989d1bb8d0c3., pid=51, masterSystemTime=1731042806641 2024-11-08T05:13:26,865 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=87b3455cffd1f4e06e5901b3bcbed9dc, ASSIGN in 615 msec 2024-11-08T05:13:26,867 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1114): Opened 27c9aaf34aaf98cd5bc8030630ae7fe3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66602415, jitterRate=-0.007546678185462952}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T05:13:26,867 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 27c9aaf34aaf98cd5bc8030630ae7fe3 2024-11-08T05:13:26,867 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=81806cb184c70c9dc0df408694092d85, regionState=OPEN, openSeqNum=2, regionLocation=d1ba675678c9,46753,1731042802851 2024-11-08T05:13:26,868 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1006): Region open journal for 27c9aaf34aaf98cd5bc8030630ae7fe3: Running coprocessor pre-open hook at 1731042806844Writing region info on filesystem at 1731042806844Initializing all the Stores at 1731042806845 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731042806845Cleaning up temporary data from old regions at 1731042806856 (+11 ms)Running coprocessor post-open hooks at 1731042806867 (+11 ms)Region opened successfully at 1731042806867 2024-11-08T05:13:26,868 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37587 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=d1ba675678c9,46753,1731042802851, table=test, region=9b3274d80dcb7deed617c19ac1de4a68. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-08T05:13:26,868 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37587 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=d1ba675678c9,46753,1731042802851, table=test, region=445b14856951586918c91d4ba342f1ff. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-08T05:13:26,868 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37587 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=d1ba675678c9,46753,1731042802851, table=test, region=81806cb184c70c9dc0df408694092d85. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-08T05:13:26,869 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2266): Finished post open deploy task for test,hhh,1731042805591.2d23e3e11ecc480e314e989d1bb8d0c3. 2024-11-08T05:13:26,869 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(153): Opened test,hhh,1731042805591.2d23e3e11ecc480e314e989d1bb8d0c3. 2024-11-08T05:13:26,869 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(132): Open test,ooo,1731042805591.d951febea78018bb9f10f07d8788575d. 2024-11-08T05:13:26,870 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2236): Post open deploy tasks for test,mmm,1731042805591.27c9aaf34aaf98cd5bc8030630ae7fe3., pid=41, masterSystemTime=1731042806586 2024-11-08T05:13:26,869 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7752): Opening region: {ENCODED => d951febea78018bb9f10f07d8788575d, NAME => 'test,ooo,1731042805591.d951febea78018bb9f10f07d8788575d.', STARTKEY => 'ooo', ENDKEY => 'ppp'} 2024-11-08T05:13:26,870 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test d951febea78018bb9f10f07d8788575d 2024-11-08T05:13:26,870 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(898): Instantiated test,ooo,1731042805591.d951febea78018bb9f10f07d8788575d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T05:13:26,870 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7794): checking encryption for d951febea78018bb9f10f07d8788575d 2024-11-08T05:13:26,870 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7797): checking classloading for d951febea78018bb9f10f07d8788575d 2024-11-08T05:13:26,871 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure 81806cb184c70c9dc0df408694092d85, server=d1ba675678c9,46753,1731042802851 because future has completed 2024-11-08T05:13:26,873 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2266): Finished post open deploy task for test,mmm,1731042805591.27c9aaf34aaf98cd5bc8030630ae7fe3. 2024-11-08T05:13:26,873 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(153): Opened test,mmm,1731042805591.27c9aaf34aaf98cd5bc8030630ae7fe3. 2024-11-08T05:13:26,873 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(132): Open test,bbb,1731042805591.eeaad0e7e7aa66f0b51c273c0aaf953c. 2024-11-08T05:13:26,874 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7752): Opening region: {ENCODED => eeaad0e7e7aa66f0b51c273c0aaf953c, NAME => 'test,bbb,1731042805591.eeaad0e7e7aa66f0b51c273c0aaf953c.', STARTKEY => 'bbb', ENDKEY => 'ccc'} 2024-11-08T05:13:26,874 INFO [StoreOpener-d951febea78018bb9f10f07d8788575d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region d951febea78018bb9f10f07d8788575d 2024-11-08T05:13:26,874 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test eeaad0e7e7aa66f0b51c273c0aaf953c 2024-11-08T05:13:26,874 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(898): Instantiated test,bbb,1731042805591.eeaad0e7e7aa66f0b51c273c0aaf953c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T05:13:26,875 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7794): checking encryption for eeaad0e7e7aa66f0b51c273c0aaf953c 2024-11-08T05:13:26,875 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7797): checking classloading for eeaad0e7e7aa66f0b51c273c0aaf953c 2024-11-08T05:13:26,876 INFO [StoreOpener-d951febea78018bb9f10f07d8788575d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d951febea78018bb9f10f07d8788575d columnFamilyName family 2024-11-08T05:13:26,876 DEBUG [StoreOpener-d951febea78018bb9f10f07d8788575d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T05:13:26,877 INFO [StoreOpener-d951febea78018bb9f10f07d8788575d-1 {}] regionserver.HStore(327): Store=d951febea78018bb9f10f07d8788575d/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T05:13:26,877 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1038): replaying wal for d951febea78018bb9f10f07d8788575d 2024-11-08T05:13:26,878 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/default/test/d951febea78018bb9f10f07d8788575d 2024-11-08T05:13:26,878 INFO [StoreOpener-eeaad0e7e7aa66f0b51c273c0aaf953c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region eeaad0e7e7aa66f0b51c273c0aaf953c 2024-11-08T05:13:26,878 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=27 2024-11-08T05:13:26,878 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=27, state=SUCCESS, hasLock=false; OpenRegionProcedure 81806cb184c70c9dc0df408694092d85, server=d1ba675678c9,46753,1731042802851 in 414 msec 2024-11-08T05:13:26,879 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/default/test/d951febea78018bb9f10f07d8788575d 2024-11-08T05:13:26,879 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1048): stopping wal replay for d951febea78018bb9f10f07d8788575d 2024-11-08T05:13:26,879 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1060): Cleaning up temporary data for d951febea78018bb9f10f07d8788575d 2024-11-08T05:13:26,880 INFO [StoreOpener-eeaad0e7e7aa66f0b51c273c0aaf953c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eeaad0e7e7aa66f0b51c273c0aaf953c columnFamilyName family 2024-11-08T05:13:26,880 DEBUG [StoreOpener-eeaad0e7e7aa66f0b51c273c0aaf953c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T05:13:26,881 INFO [StoreOpener-eeaad0e7e7aa66f0b51c273c0aaf953c-1 {}] regionserver.HStore(327): Store=eeaad0e7e7aa66f0b51c273c0aaf953c/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T05:13:26,881 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1038): replaying wal for eeaad0e7e7aa66f0b51c273c0aaf953c 2024-11-08T05:13:26,882 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1093): writing seq id for d951febea78018bb9f10f07d8788575d 2024-11-08T05:13:26,883 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=81806cb184c70c9dc0df408694092d85, ASSIGN in 635 msec 2024-11-08T05:13:26,883 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/default/test/eeaad0e7e7aa66f0b51c273c0aaf953c 2024-11-08T05:13:26,884 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/default/test/eeaad0e7e7aa66f0b51c273c0aaf953c 2024-11-08T05:13:26,885 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=27c9aaf34aaf98cd5bc8030630ae7fe3, regionState=OPEN, openSeqNum=2, regionLocation=d1ba675678c9,41485,1731042802970 2024-11-08T05:13:26,885 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1048): stopping wal replay for eeaad0e7e7aa66f0b51c273c0aaf953c 2024-11-08T05:13:26,885 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1060): Cleaning up temporary data for eeaad0e7e7aa66f0b51c273c0aaf953c 2024-11-08T05:13:26,886 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/default/test/d951febea78018bb9f10f07d8788575d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T05:13:26,887 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1114): Opened d951febea78018bb9f10f07d8788575d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65281202, jitterRate=-0.027234286069869995}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T05:13:26,887 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d951febea78018bb9f10f07d8788575d 2024-11-08T05:13:26,887 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1006): Region open journal for d951febea78018bb9f10f07d8788575d: Running coprocessor pre-open hook at 1731042806871Writing region info on filesystem at 1731042806871Initializing all the Stores at 1731042806873 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731042806873Cleaning up temporary data from old regions at 1731042806879 (+6 ms)Running coprocessor post-open hooks at 1731042806887 (+8 ms)Region opened successfully at 1731042806887 2024-11-08T05:13:26,888 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1093): writing seq id for eeaad0e7e7aa66f0b51c273c0aaf953c 2024-11-08T05:13:26,889 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=2d23e3e11ecc480e314e989d1bb8d0c3, regionState=OPEN, openSeqNum=2, regionLocation=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:26,892 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/default/test/eeaad0e7e7aa66f0b51c273c0aaf953c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T05:13:26,893 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ooo,1731042805591.d951febea78018bb9f10f07d8788575d., pid=52, masterSystemTime=1731042806641 2024-11-08T05:13:26,893 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1114): Opened eeaad0e7e7aa66f0b51c273c0aaf953c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63666325, jitterRate=-0.05129782855510712}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T05:13:26,894 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1122): Running coprocessor post-open hooks for eeaad0e7e7aa66f0b51c273c0aaf953c 2024-11-08T05:13:26,894 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1006): Region open journal for eeaad0e7e7aa66f0b51c273c0aaf953c: Running coprocessor pre-open hook at 1731042806875Writing region info on filesystem at 1731042806875Initializing all the Stores at 1731042806876 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731042806877 (+1 ms)Cleaning up temporary data from old regions at 1731042806885 (+8 ms)Running coprocessor post-open hooks at 1731042806894 (+9 ms)Region opened successfully at 1731042806894 2024-11-08T05:13:26,894 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure 27c9aaf34aaf98cd5bc8030630ae7fe3, server=d1ba675678c9,41485,1731042802970 because future has completed 2024-11-08T05:13:26,896 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ooo,1731042805591.d951febea78018bb9f10f07d8788575d. 2024-11-08T05:13:26,896 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2236): Post open deploy tasks for test,bbb,1731042805591.eeaad0e7e7aa66f0b51c273c0aaf953c., pid=44, masterSystemTime=1731042806586 2024-11-08T05:13:26,896 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(153): Opened test,ooo,1731042805591.d951febea78018bb9f10f07d8788575d. 2024-11-08T05:13:26,896 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(132): Open test,lll,1731042805591.40440b5f39bb5421276aa0b156481a9e. 2024-11-08T05:13:26,896 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7752): Opening region: {ENCODED => 40440b5f39bb5421276aa0b156481a9e, NAME => 'test,lll,1731042805591.40440b5f39bb5421276aa0b156481a9e.', STARTKEY => 'lll', ENDKEY => 'mmm'} 2024-11-08T05:13:26,896 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=51, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2d23e3e11ecc480e314e989d1bb8d0c3, server=d1ba675678c9,39291,1731042803020 because future has completed 2024-11-08T05:13:26,896 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 40440b5f39bb5421276aa0b156481a9e 2024-11-08T05:13:26,897 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(898): Instantiated test,lll,1731042805591.40440b5f39bb5421276aa0b156481a9e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T05:13:26,897 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7794): checking encryption for 40440b5f39bb5421276aa0b156481a9e 2024-11-08T05:13:26,897 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7797): checking classloading for 40440b5f39bb5421276aa0b156481a9e 2024-11-08T05:13:26,900 INFO [StoreOpener-40440b5f39bb5421276aa0b156481a9e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 40440b5f39bb5421276aa0b156481a9e 2024-11-08T05:13:26,901 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2266): Finished post open deploy task for test,bbb,1731042805591.eeaad0e7e7aa66f0b51c273c0aaf953c. 2024-11-08T05:13:26,901 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(153): Opened test,bbb,1731042805591.eeaad0e7e7aa66f0b51c273c0aaf953c. 2024-11-08T05:13:26,901 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(132): Open test,kkk,1731042805591.0e1c63297d3a8aea2fdbd1842301c811. 2024-11-08T05:13:26,901 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=26 2024-11-08T05:13:26,901 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=26, state=SUCCESS, hasLock=false; OpenRegionProcedure 7c8f72849b2e1719ed903afc73e0571a, server=d1ba675678c9,41485,1731042802970 in 424 msec 2024-11-08T05:13:26,902 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7752): Opening region: {ENCODED => 0e1c63297d3a8aea2fdbd1842301c811, NAME => 'test,kkk,1731042805591.0e1c63297d3a8aea2fdbd1842301c811.', STARTKEY => 'kkk', ENDKEY => 'lll'} 2024-11-08T05:13:26,902 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 0e1c63297d3a8aea2fdbd1842301c811 2024-11-08T05:13:26,902 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(898): Instantiated test,kkk,1731042805591.0e1c63297d3a8aea2fdbd1842301c811.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T05:13:26,903 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7794): checking encryption for 0e1c63297d3a8aea2fdbd1842301c811 2024-11-08T05:13:26,903 INFO [StoreOpener-40440b5f39bb5421276aa0b156481a9e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 40440b5f39bb5421276aa0b156481a9e columnFamilyName family 2024-11-08T05:13:26,903 DEBUG [StoreOpener-40440b5f39bb5421276aa0b156481a9e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T05:13:26,903 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7797): checking classloading for 0e1c63297d3a8aea2fdbd1842301c811 2024-11-08T05:13:26,904 INFO [StoreOpener-40440b5f39bb5421276aa0b156481a9e-1 {}] regionserver.HStore(327): Store=40440b5f39bb5421276aa0b156481a9e/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T05:13:26,904 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1038): replaying wal for 40440b5f39bb5421276aa0b156481a9e 2024-11-08T05:13:26,905 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=7c8f72849b2e1719ed903afc73e0571a, ASSIGN in 658 msec 2024-11-08T05:13:26,905 INFO [StoreOpener-0e1c63297d3a8aea2fdbd1842301c811-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 0e1c63297d3a8aea2fdbd1842301c811 2024-11-08T05:13:26,906 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/default/test/40440b5f39bb5421276aa0b156481a9e 2024-11-08T05:13:26,906 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/default/test/40440b5f39bb5421276aa0b156481a9e 2024-11-08T05:13:26,907 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=eeaad0e7e7aa66f0b51c273c0aaf953c, regionState=OPEN, openSeqNum=2, regionLocation=d1ba675678c9,41485,1731042802970 2024-11-08T05:13:26,907 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1048): stopping wal replay for 40440b5f39bb5421276aa0b156481a9e 2024-11-08T05:13:26,907 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1060): Cleaning up temporary data for 40440b5f39bb5421276aa0b156481a9e 2024-11-08T05:13:26,909 INFO [StoreOpener-0e1c63297d3a8aea2fdbd1842301c811-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0e1c63297d3a8aea2fdbd1842301c811 columnFamilyName family 2024-11-08T05:13:26,909 DEBUG [StoreOpener-0e1c63297d3a8aea2fdbd1842301c811-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T05:13:26,910 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=44, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure eeaad0e7e7aa66f0b51c273c0aaf953c, server=d1ba675678c9,41485,1731042802970 because future has completed 2024-11-08T05:13:26,910 INFO [StoreOpener-0e1c63297d3a8aea2fdbd1842301c811-1 {}] regionserver.HStore(327): Store=0e1c63297d3a8aea2fdbd1842301c811/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T05:13:26,911 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1093): writing seq id for 40440b5f39bb5421276aa0b156481a9e 2024-11-08T05:13:26,911 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1038): replaying wal for 0e1c63297d3a8aea2fdbd1842301c811 2024-11-08T05:13:26,913 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/default/test/0e1c63297d3a8aea2fdbd1842301c811 2024-11-08T05:13:26,914 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/default/test/0e1c63297d3a8aea2fdbd1842301c811 2024-11-08T05:13:26,915 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1048): stopping wal replay for 0e1c63297d3a8aea2fdbd1842301c811 2024-11-08T05:13:26,915 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1060): Cleaning up temporary data for 0e1c63297d3a8aea2fdbd1842301c811 2024-11-08T05:13:26,916 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/default/test/40440b5f39bb5421276aa0b156481a9e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T05:13:26,916 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=12 2024-11-08T05:13:26,917 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=12, state=SUCCESS, hasLock=false; OpenRegionProcedure 2d23e3e11ecc480e314e989d1bb8d0c3, server=d1ba675678c9,39291,1731042803020 in 376 msec 2024-11-08T05:13:26,917 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1114): Opened 40440b5f39bb5421276aa0b156481a9e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73346092, jitterRate=0.09294193983078003}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T05:13:26,917 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 40440b5f39bb5421276aa0b156481a9e 2024-11-08T05:13:26,917 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1006): Region open journal for 40440b5f39bb5421276aa0b156481a9e: Running coprocessor pre-open hook at 1731042806897Writing region info on filesystem at 1731042806897Initializing all the Stores at 1731042806899 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731042806899Cleaning up temporary data from old regions at 1731042806907 (+8 ms)Running coprocessor post-open hooks at 1731042806917 (+10 ms)Region opened successfully at 1731042806917 2024-11-08T05:13:26,918 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2236): Post open deploy tasks for test,lll,1731042805591.40440b5f39bb5421276aa0b156481a9e., pid=50, masterSystemTime=1731042806641 2024-11-08T05:13:26,920 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1093): writing seq id for 0e1c63297d3a8aea2fdbd1842301c811 2024-11-08T05:13:26,920 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=2d23e3e11ecc480e314e989d1bb8d0c3, ASSIGN in 673 msec 2024-11-08T05:13:26,922 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2266): Finished post open deploy task for test,lll,1731042805591.40440b5f39bb5421276aa0b156481a9e. 2024-11-08T05:13:26,923 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(153): Opened test,lll,1731042805591.40440b5f39bb5421276aa0b156481a9e. 2024-11-08T05:13:26,923 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(132): Open test,eee,1731042805591.d7680a9929f15640ca61551f186f33c1. 2024-11-08T05:13:26,923 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7752): Opening region: {ENCODED => d7680a9929f15640ca61551f186f33c1, NAME => 'test,eee,1731042805591.d7680a9929f15640ca61551f186f33c1.', STARTKEY => 'eee', ENDKEY => 'fff'} 2024-11-08T05:13:26,924 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test d7680a9929f15640ca61551f186f33c1 2024-11-08T05:13:26,924 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(898): Instantiated test,eee,1731042805591.d7680a9929f15640ca61551f186f33c1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T05:13:26,924 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7794): checking encryption for d7680a9929f15640ca61551f186f33c1 2024-11-08T05:13:26,924 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7797): checking classloading for d7680a9929f15640ca61551f186f33c1 2024-11-08T05:13:26,924 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/default/test/0e1c63297d3a8aea2fdbd1842301c811/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T05:13:26,926 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1114): Opened 0e1c63297d3a8aea2fdbd1842301c811; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62297690, jitterRate=-0.0716920793056488}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T05:13:26,926 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0e1c63297d3a8aea2fdbd1842301c811 2024-11-08T05:13:26,926 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1006): Region open journal for 0e1c63297d3a8aea2fdbd1842301c811: Running coprocessor pre-open hook at 1731042806904Writing region info on filesystem at 1731042806904Initializing all the Stores at 1731042806905 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731042806905Cleaning up temporary data from old regions at 1731042806915 (+10 ms)Running coprocessor post-open hooks at 1731042806926 (+11 ms)Region opened successfully at 1731042806926 2024-11-08T05:13:26,927 INFO [StoreOpener-d7680a9929f15640ca61551f186f33c1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region d7680a9929f15640ca61551f186f33c1 2024-11-08T05:13:26,927 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2236): Post open deploy tasks for test,kkk,1731042805591.0e1c63297d3a8aea2fdbd1842301c811., pid=46, masterSystemTime=1731042806586 2024-11-08T05:13:26,929 INFO [StoreOpener-d7680a9929f15640ca61551f186f33c1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d7680a9929f15640ca61551f186f33c1 columnFamilyName family 2024-11-08T05:13:26,930 DEBUG [StoreOpener-d7680a9929f15640ca61551f186f33c1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T05:13:26,931 INFO [StoreOpener-d7680a9929f15640ca61551f186f33c1-1 {}] regionserver.HStore(327): Store=d7680a9929f15640ca61551f186f33c1/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T05:13:26,931 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1038): replaying wal for d7680a9929f15640ca61551f186f33c1 2024-11-08T05:13:26,931 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=44, resume processing ppid=6 2024-11-08T05:13:26,931 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, ppid=6, state=SUCCESS, hasLock=false; OpenRegionProcedure eeaad0e7e7aa66f0b51c273c0aaf953c, server=d1ba675678c9,41485,1731042802970 in 429 msec 2024-11-08T05:13:26,932 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2266): Finished post open deploy task for test,kkk,1731042805591.0e1c63297d3a8aea2fdbd1842301c811. 2024-11-08T05:13:26,932 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(153): Opened test,kkk,1731042805591.0e1c63297d3a8aea2fdbd1842301c811. 2024-11-08T05:13:26,933 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/default/test/d7680a9929f15640ca61551f186f33c1 2024-11-08T05:13:26,934 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/default/test/d7680a9929f15640ca61551f186f33c1 2024-11-08T05:13:26,934 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=eeaad0e7e7aa66f0b51c273c0aaf953c, ASSIGN in 688 msec 2024-11-08T05:13:26,935 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1048): stopping wal replay for d7680a9929f15640ca61551f186f33c1 2024-11-08T05:13:26,935 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1060): Cleaning up temporary data for d7680a9929f15640ca61551f186f33c1 2024-11-08T05:13:26,936 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=0e1c63297d3a8aea2fdbd1842301c811, regionState=OPEN, openSeqNum=2, regionLocation=d1ba675678c9,41485,1731042802970 2024-11-08T05:13:26,938 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1093): writing seq id for d7680a9929f15640ca61551f186f33c1 2024-11-08T05:13:26,941 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37587 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=d1ba675678c9,39291,1731042803020, table=test, region=6240e38715889f3bb77bb89478c81a6c. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-08T05:13:26,941 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37587 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=d1ba675678c9,39291,1731042803020, table=test, region=12f1148a7b8db3e4f515337b8b7746fa. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-08T05:13:26,944 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37587 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=d1ba675678c9,41485,1731042802970, table=test, region=65d5312dbd2f96bf3b16bdfd7504dff6. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-08T05:13:26,945 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37587 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=d1ba675678c9,41485,1731042802970, table=test, region=0e1c63297d3a8aea2fdbd1842301c811. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-08T05:13:26,945 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37587 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=d1ba675678c9,41485,1731042802970, table=test, region=27c9aaf34aaf98cd5bc8030630ae7fe3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-08T05:13:26,945 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=46, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0e1c63297d3a8aea2fdbd1842301c811, server=d1ba675678c9,41485,1731042802970 because future has completed 2024-11-08T05:13:26,947 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=40440b5f39bb5421276aa0b156481a9e, regionState=OPEN, openSeqNum=2, regionLocation=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:26,951 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=50, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 40440b5f39bb5421276aa0b156481a9e, server=d1ba675678c9,39291,1731042803020 because future has completed 2024-11-08T05:13:26,953 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/default/test/d7680a9929f15640ca61551f186f33c1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T05:13:26,954 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1114): Opened d7680a9929f15640ca61551f186f33c1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68504875, jitterRate=0.02080218493938446}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T05:13:26,955 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d7680a9929f15640ca61551f186f33c1 2024-11-08T05:13:26,955 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1006): Region open journal for d7680a9929f15640ca61551f186f33c1: Running coprocessor pre-open hook at 1731042806924Writing region info on filesystem at 1731042806924Initializing all the Stores at 1731042806926 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731042806926Cleaning up temporary data from old regions at 1731042806935 (+9 ms)Running coprocessor post-open hooks at 1731042806955 (+20 ms)Region opened successfully at 1731042806955 2024-11-08T05:13:26,957 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2236): Post open deploy tasks for test,eee,1731042805591.d7680a9929f15640ca61551f186f33c1., pid=54, masterSystemTime=1731042806641 2024-11-08T05:13:26,961 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=46, resume processing ppid=15 2024-11-08T05:13:26,962 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, ppid=15, state=SUCCESS, hasLock=false; OpenRegionProcedure 0e1c63297d3a8aea2fdbd1842301c811, server=d1ba675678c9,41485,1731042802970 in 448 msec 2024-11-08T05:13:26,962 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2266): Finished post open deploy task for test,eee,1731042805591.d7680a9929f15640ca61551f186f33c1. 2024-11-08T05:13:26,962 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(153): Opened test,eee,1731042805591.d7680a9929f15640ca61551f186f33c1. 2024-11-08T05:13:26,963 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(132): Open test,ccc,1731042805591.68bbd0353f97ee12228f7798b0b987ea. 2024-11-08T05:13:26,963 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7752): Opening region: {ENCODED => 68bbd0353f97ee12228f7798b0b987ea, NAME => 'test,ccc,1731042805591.68bbd0353f97ee12228f7798b0b987ea.', STARTKEY => 'ccc', ENDKEY => 'ddd'} 2024-11-08T05:13:26,963 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 68bbd0353f97ee12228f7798b0b987ea 2024-11-08T05:13:26,964 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(898): Instantiated test,ccc,1731042805591.68bbd0353f97ee12228f7798b0b987ea.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T05:13:26,964 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=d7680a9929f15640ca61551f186f33c1, regionState=OPEN, openSeqNum=2, regionLocation=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:26,964 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7794): checking encryption for 68bbd0353f97ee12228f7798b0b987ea 2024-11-08T05:13:26,964 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7797): checking classloading for 68bbd0353f97ee12228f7798b0b987ea 2024-11-08T05:13:26,972 INFO [StoreOpener-68bbd0353f97ee12228f7798b0b987ea-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 68bbd0353f97ee12228f7798b0b987ea 2024-11-08T05:13:26,973 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=0e1c63297d3a8aea2fdbd1842301c811, ASSIGN in 723 msec 2024-11-08T05:13:26,974 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=54, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure d7680a9929f15640ca61551f186f33c1, server=d1ba675678c9,39291,1731042803020 because future has completed 2024-11-08T05:13:26,974 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37587 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=d1ba675678c9,46753,1731042802851, table=test, region=9b3274d80dcb7deed617c19ac1de4a68. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-08T05:13:26,974 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37587 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=d1ba675678c9,46753,1731042802851, table=test, region=445b14856951586918c91d4ba342f1ff. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-08T05:13:26,975 INFO [StoreOpener-68bbd0353f97ee12228f7798b0b987ea-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 68bbd0353f97ee12228f7798b0b987ea columnFamilyName family 2024-11-08T05:13:26,975 DEBUG [StoreOpener-68bbd0353f97ee12228f7798b0b987ea-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T05:13:26,976 INFO [StoreOpener-68bbd0353f97ee12228f7798b0b987ea-1 {}] regionserver.HStore(327): Store=68bbd0353f97ee12228f7798b0b987ea/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T05:13:26,976 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1038): replaying wal for 68bbd0353f97ee12228f7798b0b987ea 2024-11-08T05:13:26,978 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/default/test/68bbd0353f97ee12228f7798b0b987ea 2024-11-08T05:13:26,978 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/default/test/68bbd0353f97ee12228f7798b0b987ea 2024-11-08T05:13:26,979 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1048): stopping wal replay for 68bbd0353f97ee12228f7798b0b987ea 2024-11-08T05:13:26,979 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1060): Cleaning up temporary data for 68bbd0353f97ee12228f7798b0b987ea 2024-11-08T05:13:26,983 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1093): writing seq id for 68bbd0353f97ee12228f7798b0b987ea 2024-11-08T05:13:26,984 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=50, resume processing ppid=16 2024-11-08T05:13:26,984 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure 40440b5f39bb5421276aa0b156481a9e, server=d1ba675678c9,39291,1731042803020 in 446 msec 2024-11-08T05:13:26,992 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=40440b5f39bb5421276aa0b156481a9e, ASSIGN in 740 msec 2024-11-08T05:13:26,999 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=54, resume processing ppid=9 2024-11-08T05:13:26,999 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, ppid=9, state=SUCCESS, hasLock=false; OpenRegionProcedure d7680a9929f15640ca61551f186f33c1, server=d1ba675678c9,39291,1731042803020 in 439 msec 2024-11-08T05:13:26,999 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/default/test/68bbd0353f97ee12228f7798b0b987ea/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T05:13:27,000 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1114): Opened 68bbd0353f97ee12228f7798b0b987ea; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59369666, jitterRate=-0.1153230369091034}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T05:13:27,001 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 68bbd0353f97ee12228f7798b0b987ea 2024-11-08T05:13:27,001 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1006): Region open journal for 68bbd0353f97ee12228f7798b0b987ea: Running coprocessor pre-open hook at 1731042806964Writing region info on filesystem at 1731042806964Initializing all the Stores at 1731042806971 (+7 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731042806971Cleaning up temporary data from old regions at 1731042806979 (+8 ms)Running coprocessor post-open hooks at 1731042807001 (+22 ms)Region opened successfully at 1731042807001 2024-11-08T05:13:27,002 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ccc,1731042805591.68bbd0353f97ee12228f7798b0b987ea., pid=56, masterSystemTime=1731042806641 2024-11-08T05:13:27,004 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=d7680a9929f15640ca61551f186f33c1, ASSIGN in 755 msec 2024-11-08T05:13:27,008 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=d951febea78018bb9f10f07d8788575d, regionState=OPEN, openSeqNum=2, regionLocation=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:27,009 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ccc,1731042805591.68bbd0353f97ee12228f7798b0b987ea. 2024-11-08T05:13:27,009 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(153): Opened test,ccc,1731042805591.68bbd0353f97ee12228f7798b0b987ea. 2024-11-08T05:13:27,012 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=52, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure d951febea78018bb9f10f07d8788575d, server=d1ba675678c9,39291,1731042803020 because future has completed 2024-11-08T05:13:27,013 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=68bbd0353f97ee12228f7798b0b987ea, regionState=OPEN, openSeqNum=2, regionLocation=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:27,018 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 68bbd0353f97ee12228f7798b0b987ea, server=d1ba675678c9,39291,1731042803020 because future has completed 2024-11-08T05:13:27,027 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=7 2024-11-08T05:13:27,027 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 68bbd0353f97ee12228f7798b0b987ea, server=d1ba675678c9,39291,1731042803020 in 458 msec 2024-11-08T05:13:27,036 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=68bbd0353f97ee12228f7798b0b987ea, ASSIGN in 783 msec 2024-11-08T05:13:27,046 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37587 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=d1ba675678c9,39291,1731042803020, table=test, region=6240e38715889f3bb77bb89478c81a6c. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-08T05:13:27,046 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37587 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=d1ba675678c9,39291,1731042803020, table=test, region=12f1148a7b8db3e4f515337b8b7746fa. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-08T05:13:27,050 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37587 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=d1ba675678c9,41485,1731042802970, table=test, region=65d5312dbd2f96bf3b16bdfd7504dff6. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-08T05:13:27,051 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37587 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=d1ba675678c9,41485,1731042802970, table=test, region=27c9aaf34aaf98cd5bc8030630ae7fe3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-08T05:13:27,053 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=52, resume processing ppid=19 2024-11-08T05:13:27,053 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, ppid=19, state=SUCCESS, hasLock=false; OpenRegionProcedure d951febea78018bb9f10f07d8788575d, server=d1ba675678c9,39291,1731042803020 in 499 msec 2024-11-08T05:13:27,064 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=d951febea78018bb9f10f07d8788575d, ASSIGN in 809 msec 2024-11-08T05:13:27,080 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37587 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=d1ba675678c9,46753,1731042802851, table=test, region=9b3274d80dcb7deed617c19ac1de4a68. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-08T05:13:27,081 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37587 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=d1ba675678c9,46753,1731042802851, table=test, region=445b14856951586918c91d4ba342f1ff. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-08T05:13:27,084 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=17 2024-11-08T05:13:27,084 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=17, state=SUCCESS, hasLock=false; OpenRegionProcedure 27c9aaf34aaf98cd5bc8030630ae7fe3, server=d1ba675678c9,41485,1731042802970 in 596 msec 2024-11-08T05:13:27,088 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=27c9aaf34aaf98cd5bc8030630ae7fe3, ASSIGN in 840 msec 2024-11-08T05:13:27,093 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=24 2024-11-08T05:13:27,093 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=24, state=SUCCESS, hasLock=false; OpenRegionProcedure 445b14856951586918c91d4ba342f1ff, server=d1ba675678c9,46753,1731042802851 in 634 msec 2024-11-08T05:13:27,097 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=445b14856951586918c91d4ba342f1ff, ASSIGN in 850 msec 2024-11-08T05:13:27,106 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=5 2024-11-08T05:13:27,106 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 6240e38715889f3bb77bb89478c81a6c, server=d1ba675678c9,39291,1731042803020 in 540 msec 2024-11-08T05:13:27,108 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=6240e38715889f3bb77bb89478c81a6c, ASSIGN in 862 msec 2024-11-08T05:13:27,118 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=11 2024-11-08T05:13:27,118 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure 9b3274d80dcb7deed617c19ac1de4a68, server=d1ba675678c9,46753,1731042802851 in 684 msec 2024-11-08T05:13:27,128 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=9b3274d80dcb7deed617c19ac1de4a68, ASSIGN in 874 msec 2024-11-08T05:13:27,141 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=22 2024-11-08T05:13:27,141 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=22, state=SUCCESS, hasLock=false; OpenRegionProcedure 12f1148a7b8db3e4f515337b8b7746fa, server=d1ba675678c9,39291,1731042803020 in 588 msec 2024-11-08T05:13:27,149 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=12f1148a7b8db3e4f515337b8b7746fa, ASSIGN in 897 msec 2024-11-08T05:13:27,156 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=42, resume processing ppid=10 2024-11-08T05:13:27,156 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 65d5312dbd2f96bf3b16bdfd7504dff6, server=d1ba675678c9,41485,1731042802970 in 670 msec 2024-11-08T05:13:27,161 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=4 2024-11-08T05:13:27,161 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=65d5312dbd2f96bf3b16bdfd7504dff6, ASSIGN in 912 msec 2024-11-08T05:13:27,164 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T05:13:27,164 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731042807164"}]},"ts":"1731042807164"} 2024-11-08T05:13:27,168 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=test, state=ENABLED in hbase:meta 2024-11-08T05:13:27,169 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T05:13:27,176 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test in 1.5660 sec 2024-11-08T05:13:27,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37587 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-08T05:13:27,770 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table test get assigned. Timeout = 60000ms 2024-11-08T05:13:27,770 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:test completed 2024-11-08T05:13:27,771 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T05:13:27,795 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table test assigned to meta. Checking AM states. 2024-11-08T05:13:27,795 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T05:13:27,798 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table test assigned. 2024-11-08T05:13:27,799 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T05:13:27,805 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test,, stopping at row=test ,, for max=2147483647 with caching=100 2024-11-08T05:13:27,833 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test,, stopping at row=test ,, for max=2147483647 with caching=100 2024-11-08T05:13:27,851 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T05:13:27,853 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34696, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T05:13:27,856 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T05:13:27,858 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46712, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T05:13:27,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37587 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 6240e38715889f3bb77bb89478c81a6c changed from -1.0 to 0.0, refreshing cache 2024-11-08T05:13:27,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37587 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 68bbd0353f97ee12228f7798b0b987ea changed from -1.0 to 0.0, refreshing cache 2024-11-08T05:13:27,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37587 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region d7680a9929f15640ca61551f186f33c1 changed from -1.0 to 0.0, refreshing cache 2024-11-08T05:13:27,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37587 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 2d23e3e11ecc480e314e989d1bb8d0c3 changed from -1.0 to 0.0, refreshing cache 2024-11-08T05:13:27,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37587 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 40440b5f39bb5421276aa0b156481a9e changed from -1.0 to 0.0, refreshing cache 2024-11-08T05:13:27,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37587 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region d951febea78018bb9f10f07d8788575d changed from -1.0 to 0.0, refreshing cache 2024-11-08T05:13:27,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37587 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 12f1148a7b8db3e4f515337b8b7746fa changed from -1.0 to 0.0, refreshing cache 2024-11-08T05:13:27,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37587 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 14efcc1b9822cc74f4d4a96648555167 changed from -1.0 to 0.0, refreshing cache 2024-11-08T05:13:27,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37587 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 87b3455cffd1f4e06e5901b3bcbed9dc changed from -1.0 to 0.0, refreshing cache 2024-11-08T05:13:27,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37587 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region eeaad0e7e7aa66f0b51c273c0aaf953c changed from -1.0 to 0.0, refreshing cache 2024-11-08T05:13:27,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37587 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 65d5312dbd2f96bf3b16bdfd7504dff6 changed from -1.0 to 0.0, refreshing cache 2024-11-08T05:13:27,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37587 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region fe1545b16a81758dcb9430dbf98c4ab9 changed from -1.0 to 0.0, refreshing cache 2024-11-08T05:13:27,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37587 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 0e1c63297d3a8aea2fdbd1842301c811 changed from -1.0 to 0.0, refreshing cache 2024-11-08T05:13:27,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37587 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 27c9aaf34aaf98cd5bc8030630ae7fe3 changed from -1.0 to 0.0, refreshing cache 2024-11-08T05:13:27,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37587 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region b0ebd9a9794acc56e430b36d1f17621c changed from -1.0 to 0.0, refreshing cache 2024-11-08T05:13:27,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37587 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region e26dcb2339b364f3699d110c15cd4bb3 changed from -1.0 to 0.0, refreshing cache 2024-11-08T05:13:27,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37587 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 7c8f72849b2e1719ed903afc73e0571a changed from -1.0 to 0.0, refreshing cache 2024-11-08T05:13:27,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37587 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region de358f808110a48fa32e0533fefdc95c changed from -1.0 to 0.0, refreshing cache 2024-11-08T05:13:27,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37587 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 64ddf686fed30cdb23b2e874efec598d changed from -1.0 to 0.0, refreshing cache 2024-11-08T05:13:27,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37587 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 9b3274d80dcb7deed617c19ac1de4a68 changed from -1.0 to 0.0, refreshing cache 2024-11-08T05:13:27,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37587 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region a12ea80f78b339cfbfc0e18b5338df0c changed from -1.0 to 0.0, refreshing cache 2024-11-08T05:13:27,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37587 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 868527ec322eba1d7cbd6371701f7ad2 changed from -1.0 to 0.0, refreshing cache 2024-11-08T05:13:27,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37587 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 7830dc6b1acea6e1d468f8008d8a82b9 changed from -1.0 to 0.0, refreshing cache 2024-11-08T05:13:27,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37587 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 445b14856951586918c91d4ba342f1ff changed from -1.0 to 0.0, refreshing cache 2024-11-08T05:13:27,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37587 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 81806cb184c70c9dc0df408694092d85 changed from -1.0 to 0.0, refreshing cache 2024-11-08T05:13:27,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37587 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region e61dc9e059921fddc5ad7f6f7971acc6 changed from -1.0 to 0.0, refreshing cache 2024-11-08T05:13:27,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37587 {}] rsgroup.RSGroupBasedLoadBalancer(138): Balancing RSGroup=default 2024-11-08T05:13:27,930 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37587 {}] rsgroup.RSGroupBasedLoadBalancer(151): Start Generate Balance plan for group: default 2024-11-08T05:13:27,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37587 {}] balancer.BaseLoadBalancer(619): Start Generate Balance plan for cluster. 2024-11-08T05:13:27,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37587 {}] balancer.BalancerClusterState(204): Hosts are {d1ba675678c9=0} racks are {/default-rack=0} 2024-11-08T05:13:27,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37587 {}] balancer.BalancerClusterState(303): server 0 has 9 regions 2024-11-08T05:13:27,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37587 {}] balancer.BalancerClusterState(303): server 1 has 9 regions 2024-11-08T05:13:27,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37587 {}] balancer.BalancerClusterState(303): server 2 has 9 regions 2024-11-08T05:13:27,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37587 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-08T05:13:27,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37587 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-08T05:13:27,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37587 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-08T05:13:27,931 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37587 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-08T05:13:27,931 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37587 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-08T05:13:27,931 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37587 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-08T05:13:27,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37587 {}] balancer.BalancerClusterState(326): Number of tables=2, number of hosts=1, number of racks=1 2024-11-08T05:13:27,937 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37587 {}] balancer.StochasticLoadBalancer(395): Cluster wide - skipping load balancing because weighted average imbalance=0.016474464579901153 <= threshold(0.025). If you want more aggressive balancing, either lower hbase.master.balancer.stochastic.minCostNeedBalance from 0.025 or increase the relative multiplier(s) of the specific cost function(s). functionCost=RegionCountSkewCostFunction : (multiplier=500.0, imbalance=0.0); PrimaryRegionCountSkewCostFunction : (not needed); MoveCostFunction : (multiplier=7.0, imbalance=0.0); ServerLocalityCostFunction : (multiplier=25.0, imbalance=0.0); RackLocalityCostFunction : (multiplier=15.0, imbalance=0.0); TableSkewCostFunction : (multiplier=35.0, imbalance=0.0); RegionReplicaHostCostFunction : (not needed); RegionReplicaRackCostFunction : (not needed); ReadRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); CPRequestCostFunction : (multiplier=5.0, imbalance=0.0); WriteRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); MemStoreSizeCostFunction : (multiplier=5.0, imbalance=0.0); StoreFileCostFunction : (multiplier=5.0, imbalance=0.0); 2024-11-08T05:13:27,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37587 {}] master.HMaster(2203): Balancer is going into sleep until next period in 300000ms 2024-11-08T05:13:27,939 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [900,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T05:13:27,947 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37587 {}] master.MasterRpcServices(567): Client=jenkins//172.17.0.3 set balanceSwitch=false 2024-11-08T05:13:27,966 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: master.assignment.TestRollbackSCP#testFailAndRollback Thread=374, OpenFileDescriptor=616, MaxFileDescriptor=1048576, SystemLoadAverage=271, ProcessCount=11, AvailableMemoryMB=7190 2024-11-08T05:13:27,969 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table hbase:meta 2024-11-08T05:13:27,969 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=hbase:meta,,1 2024-11-08T05:13:27,969 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(267): Killing d1ba675678c9,46753,1731042802851 2024-11-08T05:13:27,970 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server d1ba675678c9,46753,1731042802851: Simulated kill ***** 2024-11-08T05:13:27,970 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-08T05:13:27,972 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-08T05:13:27,974 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-08T05:13:27,975 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-08T05:13:27,975 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-08T05:13:27,975 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "Verbose": false, "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 297395712 }, "NonHeapMemoryUsage": { "committed": 151715840, "init": 7667712, "max": -1, "used": 149356536 }, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "d1ba675678c9" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "d1ba675678c9" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.Context": "regionserver", "tag.Hostname": "d1ba675678c9" } ] } 2024-11-08T05:13:27,978 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37587 {}] master.MasterRpcServices(700): d1ba675678c9,46753,1731042802851 reported a fatal error: ***** ABORTING region server d1ba675678c9,46753,1731042802851: Simulated kill ***** 2024-11-08T05:13:27,981 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'd1ba675678c9,46753,1731042802851' ***** 2024-11-08T05:13:27,981 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Simulated kill 2024-11-08T05:13:27,981 INFO [RS:0;d1ba675678c9:46753 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-08T05:13:27,982 INFO [RS:0;d1ba675678c9:46753 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-11-08T05:13:27,982 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-08T05:13:27,982 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [15,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T05:13:27,982 INFO [RS:0;d1ba675678c9:46753 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-11-08T05:13:27,982 INFO [RS:0;d1ba675678c9:46753 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-08T05:13:27,982 INFO [RS:0;d1ba675678c9:46753 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;d1ba675678c9:46753. 2024-11-08T05:13:27,982 DEBUG [RS:0;d1ba675678c9:46753 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T05:13:27,986 DEBUG [RS:0;d1ba675678c9:46753 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T05:13:27,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741836_1012 (size=32114) 2024-11-08T05:13:27,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741836_1012 (size=32114) 2024-11-08T05:13:27,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741833_1009 (size=2237) 2024-11-08T05:13:27,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741833_1009 (size=2237) 2024-11-08T05:13:27,998 DEBUG [RS:0;d1ba675678c9:46753 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T05:13:27,999 INFO [RS:0;d1ba675678c9:46753 {}] regionserver.LeaseManager(133): Closed leases 2024-11-08T05:13:27,999 INFO [RS:0;d1ba675678c9:46753 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:46753 2024-11-08T05:13:28,065 INFO [regionserver/d1ba675678c9:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-08T05:13:28,082 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37587-0x10118dcefab0000, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-08T05:13:28,082 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46753-0x10118dcefab0001, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/d1ba675678c9,46753,1731042802851 2024-11-08T05:13:28,082 INFO [RS:0;d1ba675678c9:46753 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-08T05:13:28,083 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [d1ba675678c9,46753,1731042802851] 2024-11-08T05:13:28,103 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/d1ba675678c9,46753,1731042802851 already deleted, retry=false 2024-11-08T05:13:28,103 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of d1ba675678c9,46753,1731042802851 on d1ba675678c9,37587,1731042801796 2024-11-08T05:13:28,112 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=57, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure d1ba675678c9,46753,1731042802851, splitWal=true, meta=true 2024-11-08T05:13:28,117 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1991): Scheduled ServerCrashProcedure pid=57 for d1ba675678c9,46753,1731042802851 (carryingMeta=true) d1ba675678c9,46753,1731042802851/CRASHED/regionCount=9/lock=java.util.concurrent.locks.ReentrantReadWriteLock@7e34e9c7[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-08T05:13:28,118 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=57, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure d1ba675678c9,46753,1731042802851, splitWal=true, meta=true 2024-11-08T05:13:28,119 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure d1ba675678c9,46753,1731042802851, splitWal=true, meta=true, isMeta: true 2024-11-08T05:13:28,121 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/WALs/d1ba675678c9,46753,1731042802851-splitting 2024-11-08T05:13:28,123 INFO [PEWorker-1 {}] master.SplitWALManager(105): d1ba675678c9,46753,1731042802851 WAL count=1, meta=true 2024-11-08T05:13:28,125 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure d1ba675678c9%2C46753%2C1731042802851.meta.1731042805109.meta}] 2024-11-08T05:13:28,131 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:28,133 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure d1ba675678c9%2C46753%2C1731042802851.meta.1731042805109.meta, worker=d1ba675678c9,39291,1731042803020}] 2024-11-08T05:13:28,183 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T05:13:28,192 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46753-0x10118dcefab0001, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T05:13:28,193 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46753-0x10118dcefab0001, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T05:13:28,193 INFO [RS:0;d1ba675678c9:46753 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-08T05:13:28,193 INFO [RS:0;d1ba675678c9:46753 {}] regionserver.HRegionServer(1031): Exiting; stopping=d1ba675678c9,46753,1731042802851; zookeeper connection closed. 2024-11-08T05:13:28,194 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@351080c0 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@351080c0 2024-11-08T05:13:28,292 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=59 2024-11-08T05:13:28,317 INFO [RS_LOG_REPLAY_OPS-regionserver/d1ba675678c9:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(299): Splitting hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/WALs/d1ba675678c9,46753,1731042802851-splitting/d1ba675678c9%2C46753%2C1731042802851.meta.1731042805109.meta, size=31.4 K (32114bytes) 2024-11-08T05:13:28,318 DEBUG [RS_LOG_REPLAY_OPS-regionserver/d1ba675678c9:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-11-08T05:13:28,319 DEBUG [RS_LOG_REPLAY_OPS-regionserver/d1ba675678c9:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-11-08T05:13:28,319 INFO [RS_LOG_REPLAY_OPS-regionserver/d1ba675678c9:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/WALs/d1ba675678c9,46753,1731042802851-splitting/d1ba675678c9%2C46753%2C1731042802851.meta.1731042805109.meta 2024-11-08T05:13:28,324 INFO [RS_LOG_REPLAY_OPS-regionserver/d1ba675678c9:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/WALs/d1ba675678c9,46753,1731042802851-splitting/d1ba675678c9%2C46753%2C1731042802851.meta.1731042805109.meta after 4ms 2024-11-08T05:13:28,333 INFO [RS_LOG_REPLAY_OPS-regionserver/d1ba675678c9:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(310): Open hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/WALs/d1ba675678c9,46753,1731042802851-splitting/d1ba675678c9%2C46753%2C1731042802851.meta.1731042805109.meta took 16ms 2024-11-08T05:13:28,342 DEBUG [RS_LOG_REPLAY_OPS-regionserver/d1ba675678c9:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(352): Last flushed sequenceid for 1588230740: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "info" sequence_id: 5 } store_sequence_id { family_name: "ns" sequence_id: 3 } store_sequence_id { family_name: "rep_barrier" sequence_id: 18446744073709551615 } store_sequence_id { family_name: "table" sequence_id: 6 } 2024-11-08T05:13:28,346 INFO [RS_LOG_REPLAY_OPS-regionserver/d1ba675678c9:0-0-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-d1ba675678c9%2C46753%2C1731042802851.meta.1731042805109.meta.temp 2024-11-08T05:13:28,351 INFO [RS_LOG_REPLAY_OPS-regionserver/d1ba675678c9:0-0-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-d1ba675678c9%2C46753%2C1731042802851.meta.1731042805109.meta.temp 2024-11-08T05:13:28,352 DEBUG [RS_LOG_REPLAY_OPS-regionserver/d1ba675678c9:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/WALs/d1ba675678c9,46753,1731042802851-splitting/d1ba675678c9%2C46753%2C1731042802851.meta.1731042805109.meta so closing down 2024-11-08T05:13:28,352 DEBUG [RS_LOG_REPLAY_OPS-regionserver/d1ba675678c9:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-08T05:13:28,355 INFO [RS_LOG_REPLAY_OPS-regionserver/d1ba675678c9:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.OutputSink(145): 3 split writer threads finished 2024-11-08T05:13:28,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34565 is added to blk_1073741864_1040 (size=31873) 2024-11-08T05:13:28,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741864_1040 (size=31873) 2024-11-08T05:13:28,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741864_1040 (size=31873) 2024-11-08T05:13:28,369 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-d1ba675678c9%2C46753%2C1731042802851.meta.1731042805109.meta.temp (wrote 57 edits, skipped 0 edits in 3 ms) 2024-11-08T05:13:28,371 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-d1ba675678c9%2C46753%2C1731042802851.meta.1731042805109.meta.temp to hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-08T05:13:28,374 INFO [RS_LOG_REPLAY_OPS-regionserver/d1ba675678c9:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(425): Processed 58 edits across 1 Regions in 37 ms; skipped=1; WAL=hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/WALs/d1ba675678c9,46753,1731042802851-splitting/d1ba675678c9%2C46753%2C1731042802851.meta.1731042805109.meta, size=31.4 K, length=32114, corrupted=false, cancelled=false 2024-11-08T05:13:28,374 DEBUG [RS_LOG_REPLAY_OPS-regionserver/d1ba675678c9:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(428): Completed split of hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/WALs/d1ba675678c9,46753,1731042802851-splitting/d1ba675678c9%2C46753%2C1731042802851.meta.1731042805109.meta, journal: Splitting hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/WALs/d1ba675678c9,46753,1731042802851-splitting/d1ba675678c9%2C46753%2C1731042802851.meta.1731042805109.meta, size=31.4 K (32114bytes) at 1731042808317Creating recovered edits writer path=hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-d1ba675678c9%2C46753%2C1731042802851.meta.1731042805109.meta.temp at 1731042808351 (+34 ms)Finishing writing output for hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/WALs/d1ba675678c9,46753,1731042802851-splitting/d1ba675678c9%2C46753%2C1731042802851.meta.1731042805109.meta so closing down at 1731042808352 (+1 ms)3 split writer threads finished at 1731042808355 (+3 ms)Closed recovered edits writer path=hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-d1ba675678c9%2C46753%2C1731042802851.meta.1731042805109.meta.temp (wrote 57 edits, skipped 0 edits in 3 ms) at 1731042808369 (+14 ms)Rename recovered edits hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-d1ba675678c9%2C46753%2C1731042802851.meta.1731042805109.meta.temp to hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 at 1731042808371 (+2 ms)Processed 58 edits across 1 Regions in 37 ms; skipped=1; WAL=hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/WALs/d1ba675678c9,46753,1731042802851-splitting/d1ba675678c9%2C46753%2C1731042802851.meta.1731042805109.meta, size=31.4 K, length=32114, corrupted=false, cancelled=false at 1731042808374 (+3 ms) 2024-11-08T05:13:28,375 DEBUG [RS_LOG_REPLAY_OPS-regionserver/d1ba675678c9:0-0 {event_type=RS_LOG_REPLAY, pid=59}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/WALs/d1ba675678c9,46753,1731042802851-splitting/d1ba675678c9%2C46753%2C1731042802851.meta.1731042805109.meta 2024-11-08T05:13:28,376 DEBUG [RS_LOG_REPLAY_OPS-regionserver/d1ba675678c9:0-0 {event_type=RS_LOG_REPLAY, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-11-08T05:13:28,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37587 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-11-08T05:13:28,384 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/WALs/d1ba675678c9,46753,1731042802851-splitting/d1ba675678c9%2C46753%2C1731042802851.meta.1731042805109.meta to hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/oldWALs 2024-11-08T05:13:28,387 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-08T05:13:28,387 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d1ba675678c9%2C46753%2C1731042802851.meta.1731042805109.meta, worker=d1ba675678c9,39291,1731042803020 in 252 msec 2024-11-08T05:13:28,389 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:28,392 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-11-08T05:13:28,392 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure d1ba675678c9%2C46753%2C1731042802851.meta.1731042805109.meta, worker=d1ba675678c9,39291,1731042803020 in 265 msec 2024-11-08T05:13:28,394 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/WALs/d1ba675678c9,46753,1731042802851-splitting dir is empty, no logs to split. 2024-11-08T05:13:28,394 INFO [PEWorker-1 {}] master.SplitWALManager(105): d1ba675678c9,46753,1731042802851 WAL count=0, meta=true 2024-11-08T05:13:28,394 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if d1ba675678c9,46753,1731042802851 WAL splitting is done? wals=0, meta=true 2024-11-08T05:13:28,396 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-08T05:13:28,397 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T05:13:28,398 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-08T05:13:28,427 INFO [regionserver/d1ba675678c9:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-08T05:13:28,427 INFO [regionserver/d1ba675678c9:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-08T05:13:28,548 DEBUG [d1ba675678c9:37587 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-08T05:13:28,549 DEBUG [d1ba675678c9:37587 {}] balancer.BalancerClusterState(204): Hosts are {d1ba675678c9=0} racks are {/default-rack=0} 2024-11-08T05:13:28,549 DEBUG [d1ba675678c9:37587 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-08T05:13:28,549 DEBUG [d1ba675678c9:37587 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-08T05:13:28,549 DEBUG [d1ba675678c9:37587 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-08T05:13:28,549 DEBUG [d1ba675678c9:37587 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-08T05:13:28,549 INFO [d1ba675678c9:37587 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-08T05:13:28,549 INFO [d1ba675678c9:37587 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-08T05:13:28,549 DEBUG [d1ba675678c9:37587 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-08T05:13:28,549 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=d1ba675678c9,41485,1731042802970 2024-11-08T05:13:28,551 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d1ba675678c9,41485,1731042802970, state=OPENING 2024-11-08T05:13:28,597 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39291-0x10118dcefab0003, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T05:13:28,597 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37587-0x10118dcefab0000, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T05:13:28,597 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41485-0x10118dcefab0002, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T05:13:28,597 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T05:13:28,598 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=d1ba675678c9,41485,1731042802970}] 2024-11-08T05:13:28,598 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T05:13:28,598 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T05:13:28,598 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T05:13:28,755 INFO [RS_OPEN_META-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_META, pid=61}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-08T05:13:28,755 INFO [RS_OPEN_META-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-08T05:13:28,756 INFO [RS_OPEN_META-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_META, pid=61}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-08T05:13:28,758 INFO [RS_OPEN_META-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d1ba675678c9%2C41485%2C1731042802970.meta, suffix=.meta, logDir=hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/WALs/d1ba675678c9,41485,1731042802970, archiveDir=hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/oldWALs, maxLogs=32 2024-11-08T05:13:28,771 DEBUG [RS_OPEN_META-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_META, pid=61}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/WALs/d1ba675678c9,41485,1731042802970/d1ba675678c9%2C41485%2C1731042802970.meta.1731042808759.meta, exclude list is [], retry=0 2024-11-08T05:13:28,774 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41203,DS-097c58aa-51b2-4f6f-9fb5-10702640c273,DISK] 2024-11-08T05:13:28,774 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37041,DS-7382e2ea-12bb-4310-9c38-6f88a0ed4f48,DISK] 2024-11-08T05:13:28,774 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34565,DS-580c23be-6f59-4a94-8328-84ad55cf144b,DISK] 2024-11-08T05:13:28,776 INFO [RS_OPEN_META-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/WALs/d1ba675678c9,41485,1731042802970/d1ba675678c9%2C41485%2C1731042802970.meta.1731042808759.meta 2024-11-08T05:13:28,778 DEBUG [RS_OPEN_META-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43885:43885),(127.0.0.1/127.0.0.1:35281:35281),(127.0.0.1/127.0.0.1:34771:34771)] 2024-11-08T05:13:28,778 DEBUG [RS_OPEN_META-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-08T05:13:28,778 DEBUG [RS_OPEN_META-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_META, pid=61}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-08T05:13:28,778 DEBUG [RS_OPEN_META-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-08T05:13:28,778 INFO [RS_OPEN_META-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-08T05:13:28,779 DEBUG [RS_OPEN_META-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-08T05:13:28,779 DEBUG [RS_OPEN_META-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T05:13:28,779 DEBUG [RS_OPEN_META-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-08T05:13:28,779 DEBUG [RS_OPEN_META-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-08T05:13:28,781 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-08T05:13:28,782 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-08T05:13:28,782 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T05:13:28,783 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T05:13:28,783 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-08T05:13:28,784 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-08T05:13:28,784 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T05:13:28,785 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T05:13:28,785 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-08T05:13:28,786 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-08T05:13:28,786 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T05:13:28,787 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T05:13:28,787 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-08T05:13:28,788 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-08T05:13:28,788 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T05:13:28,788 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T05:13:28,789 DEBUG [RS_OPEN_META-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-08T05:13:28,789 DEBUG [RS_OPEN_META-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/hbase/meta/1588230740 2024-11-08T05:13:28,791 DEBUG [RS_OPEN_META-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/hbase/meta/1588230740 2024-11-08T05:13:28,792 INFO [RS_OPEN_META-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-08T05:13:28,810 DEBUG [RS_OPEN_META-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5793): Applied 264, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=60, path=hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-08T05:13:28,815 INFO [RS_OPEN_META-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=28 KB heapSize=48.31 KB 2024-11-08T05:13:28,884 DEBUG [RS_OPEN_META-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_META, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/hbase/meta/1588230740/.tmp/info/a0e5e20067b44f52a4fd5336c8a77e59 is 138, key is test,bbb,1731042805591.eeaad0e7e7aa66f0b51c273c0aaf953c./info:regioninfo/1731042806907/Put/seqid=0 2024-11-08T05:13:28,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741866_1042 (size=34924) 2024-11-08T05:13:28,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34565 is added to blk_1073741866_1042 (size=34924) 2024-11-08T05:13:28,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741866_1042 (size=34924) 2024-11-08T05:13:28,896 INFO [RS_OPEN_META-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=27.86 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/hbase/meta/1588230740/.tmp/info/a0e5e20067b44f52a4fd5336c8a77e59 2024-11-08T05:13:28,965 DEBUG [RS_OPEN_META-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_META, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/hbase/meta/1588230740/.tmp/ns/fb510b5fb4584151820e72ead6f3f10e is 43, key is default/ns:d/1731042805300/Put/seqid=0 2024-11-08T05:13:28,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741867_1043 (size=5153) 2024-11-08T05:13:28,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34565 is added to blk_1073741867_1043 (size=5153) 2024-11-08T05:13:28,974 INFO [RS_OPEN_META-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/hbase/meta/1588230740/.tmp/ns/fb510b5fb4584151820e72ead6f3f10e 2024-11-08T05:13:28,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741867_1043 (size=5153) 2024-11-08T05:13:29,000 DEBUG [RS_OPEN_META-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_META, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/hbase/meta/1588230740/.tmp/table/e7270ecb7b5841cc84807192c4c79a78 is 40, key is test/table:state/1731042807164/Put/seqid=0 2024-11-08T05:13:29,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741868_1044 (size=5165) 2024-11-08T05:13:29,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34565 is added to blk_1073741868_1044 (size=5165) 2024-11-08T05:13:29,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741868_1044 (size=5165) 2024-11-08T05:13:29,011 INFO [RS_OPEN_META-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=72 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/hbase/meta/1588230740/.tmp/table/e7270ecb7b5841cc84807192c4c79a78 2024-11-08T05:13:29,022 DEBUG [RS_OPEN_META-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/hbase/meta/1588230740/.tmp/info/a0e5e20067b44f52a4fd5336c8a77e59 as hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/hbase/meta/1588230740/info/a0e5e20067b44f52a4fd5336c8a77e59 2024-11-08T05:13:29,031 INFO [RS_OPEN_META-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/hbase/meta/1588230740/info/a0e5e20067b44f52a4fd5336c8a77e59, entries=260, sequenceid=60, filesize=34.1 K 2024-11-08T05:13:29,034 DEBUG [RS_OPEN_META-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/hbase/meta/1588230740/.tmp/ns/fb510b5fb4584151820e72ead6f3f10e as hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/hbase/meta/1588230740/ns/fb510b5fb4584151820e72ead6f3f10e 2024-11-08T05:13:29,044 INFO [RS_OPEN_META-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/hbase/meta/1588230740/ns/fb510b5fb4584151820e72ead6f3f10e, entries=2, sequenceid=60, filesize=5.0 K 2024-11-08T05:13:29,045 DEBUG [RS_OPEN_META-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/hbase/meta/1588230740/.tmp/table/e7270ecb7b5841cc84807192c4c79a78 as hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/hbase/meta/1588230740/table/e7270ecb7b5841cc84807192c4c79a78 2024-11-08T05:13:29,055 INFO [RS_OPEN_META-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/hbase/meta/1588230740/table/e7270ecb7b5841cc84807192c4c79a78, entries=2, sequenceid=60, filesize=5.0 K 2024-11-08T05:13:29,061 INFO [RS_OPEN_META-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~28 KB/28672, heapSize ~48.02 KB/49168, currentSize=0 B/0 for 1588230740 in 244ms, sequenceid=60, compaction requested=false; wal=null 2024-11-08T05:13:29,062 DEBUG [RS_OPEN_META-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-08T05:13:29,066 DEBUG [RS_OPEN_META-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-08T05:13:29,067 DEBUG [RS_OPEN_META-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-08T05:13:29,067 DEBUG [RS_OPEN_META-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-08T05:13:29,068 DEBUG [RS_OPEN_META-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-08T05:13:29,070 DEBUG [RS_OPEN_META-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-08T05:13:29,072 DEBUG [RS_OPEN_META-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/hbase/meta/1588230740/recovered.edits/60.seqid, newMaxSeqId=60, maxSeqId=1 2024-11-08T05:13:29,074 INFO [RS_OPEN_META-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=61; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68117870, jitterRate=0.015035361051559448}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-08T05:13:29,074 DEBUG [RS_OPEN_META-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-08T05:13:29,074 DEBUG [RS_OPEN_META-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731042808779Writing region info on filesystem at 1731042808779Initializing all the Stores at 1731042808780 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731042808780Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731042808781 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731042808781Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731042808781Obtaining lock to block concurrent updates at 1731042808815 (+34 ms)Preparing flush snapshotting stores in 1588230740 at 1731042808815Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=28672, getHeapSize=49408, getOffHeapSize=0, getCellsCount=264 at 1731042808818 (+3 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731042808818Flushing 1588230740/info: creating writer at 1731042808820 (+2 ms)Flushing 1588230740/info: appending metadata at 1731042808874 (+54 ms)Flushing 1588230740/info: closing flushed file at 1731042808876 (+2 ms)Flushing 1588230740/ns: creating writer at 1731042808946 (+70 ms)Flushing 1588230740/ns: appending metadata at 1731042808964 (+18 ms)Flushing 1588230740/ns: closing flushed file at 1731042808964Flushing 1588230740/table: creating writer at 1731042808982 (+18 ms)Flushing 1588230740/table: appending metadata at 1731042808999 (+17 ms)Flushing 1588230740/table: closing flushed file at 1731042808999Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2193b3b4: reopening flushed file at 1731042809021 (+22 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@67ea9bbd: reopening flushed file at 1731042809032 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@23b88119: reopening flushed file at 1731042809044 (+12 ms)Finished flush of dataSize ~28 KB/28672, heapSize ~48.02 KB/49168, currentSize=0 B/0 for 1588230740 in 244ms, sequenceid=60, compaction requested=false; wal=null at 1731042809061 (+17 ms)Cleaning up temporary data from old regions at 1731042809067 (+6 ms)Running coprocessor post-open hooks at 1731042809074 (+7 ms)Region opened successfully at 1731042809074 2024-11-08T05:13:29,076 INFO [RS_OPEN_META-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=61, masterSystemTime=1731042808750 2024-11-08T05:13:29,078 DEBUG [RS_OPEN_META-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-08T05:13:29,078 INFO [RS_OPEN_META-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_META, pid=61}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-08T05:13:29,079 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=61, regionLocation=d1ba675678c9,41485,1731042802970 2024-11-08T05:13:29,080 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d1ba675678c9,41485,1731042802970, state=OPEN 2024-11-08T05:13:29,165 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37587 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=d1ba675678c9,41485,1731042802970, table=hbase:meta, region=1588230740. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-08T05:13:29,239 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39291-0x10118dcefab0003, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T05:13:29,239 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41485-0x10118dcefab0002, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T05:13:29,239 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37587-0x10118dcefab0000, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T05:13:29,240 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=61, ppid=60, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=d1ba675678c9,41485,1731042802970 2024-11-08T05:13:29,240 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T05:13:29,240 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T05:13:29,240 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T05:13:29,242 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-08T05:13:29,242 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d1ba675678c9,41485,1731042802970 in 642 msec 2024-11-08T05:13:29,246 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=60, resume processing ppid=57 2024-11-08T05:13:29,246 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 846 msec 2024-11-08T05:13:29,246 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): d1ba675678c9,46753,1731042802851 had 9 regions 2024-11-08T05:13:29,247 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure d1ba675678c9,46753,1731042802851, splitWal=true, meta=true, isMeta: false 2024-11-08T05:13:29,250 INFO [PEWorker-1 {}] master.SplitWALManager(105): d1ba675678c9,46753,1731042802851 WAL count=1, meta=false 2024-11-08T05:13:29,250 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=57, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure d1ba675678c9%2C46753%2C1731042802851.1731042804603}] 2024-11-08T05:13:29,252 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:29,254 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure d1ba675678c9%2C46753%2C1731042802851.1731042804603, worker=d1ba675678c9,39291,1731042803020}] 2024-11-08T05:13:29,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34565 is added to blk_1073741839_1015 (size=42) 2024-11-08T05:13:29,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34565 is added to blk_1073741837_1013 (size=358) 2024-11-08T05:13:29,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34565 is added to blk_1073741831_1007 (size=1321) 2024-11-08T05:13:29,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34565 is added to blk_1073741840_1016 (size=45) 2024-11-08T05:13:29,408 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=63 2024-11-08T05:13:29,425 INFO [RS_LOG_REPLAY_OPS-regionserver/d1ba675678c9:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(299): Splitting hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/WALs/d1ba675678c9,46753,1731042802851-splitting/d1ba675678c9%2C46753%2C1731042802851.1731042804603, size=2.2 K (2237bytes) 2024-11-08T05:13:29,426 INFO [RS_LOG_REPLAY_OPS-regionserver/d1ba675678c9:0-1 {event_type=RS_LOG_REPLAY, pid=63}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/WALs/d1ba675678c9,46753,1731042802851-splitting/d1ba675678c9%2C46753%2C1731042802851.1731042804603 2024-11-08T05:13:29,426 INFO [RS_LOG_REPLAY_OPS-regionserver/d1ba675678c9:0-1 {event_type=RS_LOG_REPLAY, pid=63}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/WALs/d1ba675678c9,46753,1731042802851-splitting/d1ba675678c9%2C46753%2C1731042802851.1731042804603 after 0ms 2024-11-08T05:13:29,430 INFO [RS_LOG_REPLAY_OPS-regionserver/d1ba675678c9:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(310): Open hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/WALs/d1ba675678c9,46753,1731042802851-splitting/d1ba675678c9%2C46753%2C1731042802851.1731042804603 took 5ms 2024-11-08T05:13:29,434 DEBUG [RS_LOG_REPLAY_OPS-regionserver/d1ba675678c9:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 7830dc6b1acea6e1d468f8008d8a82b9: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-08T05:13:29,436 DEBUG [RS_LOG_REPLAY_OPS-regionserver/d1ba675678c9:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 868527ec322eba1d7cbd6371701f7ad2: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-08T05:13:29,438 DEBUG [RS_LOG_REPLAY_OPS-regionserver/d1ba675678c9:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for e61dc9e059921fddc5ad7f6f7971acc6: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-08T05:13:29,440 DEBUG [RS_LOG_REPLAY_OPS-regionserver/d1ba675678c9:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for a12ea80f78b339cfbfc0e18b5338df0c: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-08T05:13:29,442 DEBUG [RS_LOG_REPLAY_OPS-regionserver/d1ba675678c9:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 64ddf686fed30cdb23b2e874efec598d: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-08T05:13:29,443 DEBUG [RS_LOG_REPLAY_OPS-regionserver/d1ba675678c9:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 9b3274d80dcb7deed617c19ac1de4a68: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-08T05:13:29,445 DEBUG [RS_LOG_REPLAY_OPS-regionserver/d1ba675678c9:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 445b14856951586918c91d4ba342f1ff: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-08T05:13:29,447 DEBUG [RS_LOG_REPLAY_OPS-regionserver/d1ba675678c9:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 81806cb184c70c9dc0df408694092d85: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-08T05:13:29,447 DEBUG [RS_LOG_REPLAY_OPS-regionserver/d1ba675678c9:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/WALs/d1ba675678c9,46753,1731042802851-splitting/d1ba675678c9%2C46753%2C1731042802851.1731042804603 so closing down 2024-11-08T05:13:29,447 DEBUG [RS_LOG_REPLAY_OPS-regionserver/d1ba675678c9:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-08T05:13:29,448 INFO [RS_LOG_REPLAY_OPS-regionserver/d1ba675678c9:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.OutputSink(145): 3 split writer threads finished 2024-11-08T05:13:29,448 INFO [RS_LOG_REPLAY_OPS-regionserver/d1ba675678c9:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(425): Processed 8 edits across 0 Regions in 18 ms; skipped=8; WAL=hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/WALs/d1ba675678c9,46753,1731042802851-splitting/d1ba675678c9%2C46753%2C1731042802851.1731042804603, size=2.2 K, length=2237, corrupted=false, cancelled=false 2024-11-08T05:13:29,448 DEBUG [RS_LOG_REPLAY_OPS-regionserver/d1ba675678c9:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(428): Completed split of hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/WALs/d1ba675678c9,46753,1731042802851-splitting/d1ba675678c9%2C46753%2C1731042802851.1731042804603, journal: Splitting hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/WALs/d1ba675678c9,46753,1731042802851-splitting/d1ba675678c9%2C46753%2C1731042802851.1731042804603, size=2.2 K (2237bytes) at 1731042809425Finishing writing output for hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/WALs/d1ba675678c9,46753,1731042802851-splitting/d1ba675678c9%2C46753%2C1731042802851.1731042804603 so closing down at 1731042809447 (+22 ms)3 split writer threads finished at 1731042809448 (+1 ms)Processed 8 edits across 0 Regions in 18 ms; skipped=8; WAL=hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/WALs/d1ba675678c9,46753,1731042802851-splitting/d1ba675678c9%2C46753%2C1731042802851.1731042804603, size=2.2 K, length=2237, corrupted=false, cancelled=false at 1731042809448 2024-11-08T05:13:29,448 DEBUG [RS_LOG_REPLAY_OPS-regionserver/d1ba675678c9:0-1 {event_type=RS_LOG_REPLAY, pid=63}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/WALs/d1ba675678c9,46753,1731042802851-splitting/d1ba675678c9%2C46753%2C1731042802851.1731042804603 2024-11-08T05:13:29,448 DEBUG [RS_LOG_REPLAY_OPS-regionserver/d1ba675678c9:0-1 {event_type=RS_LOG_REPLAY, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-11-08T05:13:29,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37587 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-11-08T05:13:29,453 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/WALs/d1ba675678c9,46753,1731042802851-splitting/d1ba675678c9%2C46753%2C1731042802851.1731042804603 to hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/oldWALs 2024-11-08T05:13:29,456 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-08T05:13:29,457 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d1ba675678c9%2C46753%2C1731042802851.1731042804603, worker=d1ba675678c9,39291,1731042803020 in 199 msec 2024-11-08T05:13:29,458 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:29,461 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=57 2024-11-08T05:13:29,461 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure d1ba675678c9%2C46753%2C1731042802851.1731042804603, worker=d1ba675678c9,39291,1731042803020 in 208 msec 2024-11-08T05:13:29,466 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/WALs/d1ba675678c9,46753,1731042802851-splitting dir is empty, no logs to split. 2024-11-08T05:13:29,466 INFO [PEWorker-1 {}] master.SplitWALManager(105): d1ba675678c9,46753,1731042802851 WAL count=0, meta=false 2024-11-08T05:13:29,466 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if d1ba675678c9,46753,1731042802851 WAL splitting is done? wals=0, meta=false 2024-11-08T05:13:29,469 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=e61dc9e059921fddc5ad7f6f7971acc6, ASSIGN}, {pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=81806cb184c70c9dc0df408694092d85, ASSIGN}, {pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=868527ec322eba1d7cbd6371701f7ad2, ASSIGN}, {pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a12ea80f78b339cfbfc0e18b5338df0c, ASSIGN}, {pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=445b14856951586918c91d4ba342f1ff, ASSIGN}, {pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=9b3274d80dcb7deed617c19ac1de4a68, ASSIGN}, {pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=7830dc6b1acea6e1d468f8008d8a82b9, ASSIGN}, {pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=64ddf686fed30cdb23b2e874efec598d, ASSIGN}] 2024-11-08T05:13:29,471 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=64ddf686fed30cdb23b2e874efec598d, ASSIGN 2024-11-08T05:13:29,472 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=64ddf686fed30cdb23b2e874efec598d, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-08T05:13:29,473 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=7830dc6b1acea6e1d468f8008d8a82b9, ASSIGN 2024-11-08T05:13:29,475 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=7830dc6b1acea6e1d468f8008d8a82b9, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-08T05:13:29,477 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=9b3274d80dcb7deed617c19ac1de4a68, ASSIGN 2024-11-08T05:13:29,478 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=9b3274d80dcb7deed617c19ac1de4a68, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-08T05:13:29,480 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=445b14856951586918c91d4ba342f1ff, ASSIGN 2024-11-08T05:13:29,482 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=445b14856951586918c91d4ba342f1ff, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-08T05:13:29,483 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a12ea80f78b339cfbfc0e18b5338df0c, ASSIGN 2024-11-08T05:13:29,484 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=a12ea80f78b339cfbfc0e18b5338df0c, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-08T05:13:29,485 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=868527ec322eba1d7cbd6371701f7ad2, ASSIGN 2024-11-08T05:13:29,486 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=868527ec322eba1d7cbd6371701f7ad2, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-08T05:13:29,487 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=81806cb184c70c9dc0df408694092d85, ASSIGN 2024-11-08T05:13:29,488 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=81806cb184c70c9dc0df408694092d85, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-08T05:13:29,489 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=e61dc9e059921fddc5ad7f6f7971acc6, ASSIGN 2024-11-08T05:13:29,490 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=e61dc9e059921fddc5ad7f6f7971acc6, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-08T05:13:29,623 DEBUG [d1ba675678c9:37587 {}] balancer.BalancerClusterState(204): Hosts are {d1ba675678c9=0} racks are {/default-rack=0} 2024-11-08T05:13:29,623 DEBUG [d1ba675678c9:37587 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-08T05:13:29,623 DEBUG [d1ba675678c9:37587 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-08T05:13:29,623 DEBUG [d1ba675678c9:37587 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-08T05:13:29,623 DEBUG [d1ba675678c9:37587 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-08T05:13:29,623 INFO [d1ba675678c9:37587 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-08T05:13:29,623 INFO [d1ba675678c9:37587 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-08T05:13:29,623 DEBUG [d1ba675678c9:37587 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-08T05:13:29,624 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=65 updating hbase:meta row=81806cb184c70c9dc0df408694092d85, regionState=OPENING, regionLocation=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:29,625 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=70 updating hbase:meta row=7830dc6b1acea6e1d468f8008d8a82b9, regionState=OPENING, regionLocation=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:29,625 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=67 updating hbase:meta row=a12ea80f78b339cfbfc0e18b5338df0c, regionState=OPENING, regionLocation=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:29,625 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server d1ba675678c9:46753 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: d1ba675678c9/172.17.0.3:46753 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T05:13:29,626 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=71 updating hbase:meta row=64ddf686fed30cdb23b2e874efec598d, regionState=OPENING, regionLocation=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:29,626 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=e61dc9e059921fddc5ad7f6f7971acc6, regionState=OPENING, regionLocation=d1ba675678c9,41485,1731042802970 2024-11-08T05:13:29,627 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=d1ba675678c9,46753,1731042802851, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=d1ba675678c9,46753,1731042802851, seqNum=-1, error=java.net.ConnectException: Call to address=d1ba675678c9:46753 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: d1ba675678c9/172.17.0.3:46753 2024-11-08T05:13:29,627 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=68 updating hbase:meta row=445b14856951586918c91d4ba342f1ff, regionState=OPENING, regionLocation=d1ba675678c9,41485,1731042802970 2024-11-08T05:13:29,627 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=d1ba675678c9,46753,1731042802851, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-08T05:13:29,627 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=66 updating hbase:meta row=868527ec322eba1d7cbd6371701f7ad2, regionState=OPENING, regionLocation=d1ba675678c9,41485,1731042802970 2024-11-08T05:13:29,627 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=d1ba675678c9,46753,1731042802851, seqNum=-1 from cache 2024-11-08T05:13:29,628 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T05:13:29,628 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d1ba675678c9,41485,1731042802970, seqNum=-1] 2024-11-08T05:13:29,628 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T05:13:29,628 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.FailedServers(52): Added failed server with address d1ba675678c9:46753 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: d1ba675678c9/172.17.0.3:46753 2024-11-08T05:13:29,628 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=69 updating hbase:meta row=9b3274d80dcb7deed617c19ac1de4a68, regionState=OPENING, regionLocation=d1ba675678c9,41485,1731042802970 2024-11-08T05:13:29,629 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server d1ba675678c9:46753 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: d1ba675678c9/172.17.0.3:46753 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T05:13:29,629 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=d1ba675678c9,46753,1731042802851, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=d1ba675678c9,41485,1731042802970, seqNum=-1, error=java.net.ConnectException: Call to address=d1ba675678c9:46753 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: d1ba675678c9/172.17.0.3:46753 2024-11-08T05:13:29,629 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36517, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T05:13:29,630 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=d1ba675678c9,46753,1731042802851, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=d1ba675678c9,41485,1731042802970, seqNum=-1, error=java.net.ConnectException: Call to address=d1ba675678c9:46753 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: d1ba675678c9/172.17.0.3:46753 2024-11-08T05:13:29,630 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=d1ba675678c9,46753,1731042802851, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=d1ba675678c9,41485,1731042802970, seqNum=-1, error=java.net.ConnectException: Call to address=d1ba675678c9:46753 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: d1ba675678c9/172.17.0.3:46753 2024-11-08T05:13:29,630 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=d1ba675678c9,46753,1731042802851, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=d1ba675678c9,41485,1731042802970, seqNum=-1, error=java.net.ConnectException: Call to address=d1ba675678c9:46753 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: d1ba675678c9/172.17.0.3:46753 2024-11-08T05:13:29,630 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=d1ba675678c9,46753,1731042802851, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=d1ba675678c9,41485,1731042802970, seqNum=-1, error=java.net.ConnectException: Call to address=d1ba675678c9:46753 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: d1ba675678c9/172.17.0.3:46753 2024-11-08T05:13:29,630 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.FailedServers(52): Added failed server with address d1ba675678c9:46753 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: d1ba675678c9/172.17.0.3:46753 2024-11-08T05:13:29,632 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=868527ec322eba1d7cbd6371701f7ad2, ASSIGN because future has completed 2024-11-08T05:13:29,633 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=72, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 868527ec322eba1d7cbd6371701f7ad2, server=d1ba675678c9,41485,1731042802970}] 2024-11-08T05:13:29,634 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=9b3274d80dcb7deed617c19ac1de4a68, ASSIGN because future has completed 2024-11-08T05:13:29,636 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9b3274d80dcb7deed617c19ac1de4a68, server=d1ba675678c9,41485,1731042802970}] 2024-11-08T05:13:29,740 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=81806cb184c70c9dc0df408694092d85, ASSIGN because future has completed 2024-11-08T05:13:29,741 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=74, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 81806cb184c70c9dc0df408694092d85, server=d1ba675678c9,39291,1731042803020}] 2024-11-08T05:13:29,741 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=7830dc6b1acea6e1d468f8008d8a82b9, ASSIGN because future has completed 2024-11-08T05:13:29,742 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=a12ea80f78b339cfbfc0e18b5338df0c, ASSIGN because future has completed 2024-11-08T05:13:29,744 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=64ddf686fed30cdb23b2e874efec598d, ASSIGN because future has completed 2024-11-08T05:13:29,745 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 64ddf686fed30cdb23b2e874efec598d, server=d1ba675678c9,39291,1731042803020}] 2024-11-08T05:13:29,745 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=e61dc9e059921fddc5ad7f6f7971acc6, ASSIGN because future has completed 2024-11-08T05:13:29,746 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=445b14856951586918c91d4ba342f1ff, ASSIGN because future has completed 2024-11-08T05:13:29,748 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 445b14856951586918c91d4ba342f1ff, server=d1ba675678c9,41485,1731042802970}] 2024-11-08T05:13:29,750 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure e61dc9e059921fddc5ad7f6f7971acc6, server=d1ba675678c9,41485,1731042802970}] 2024-11-08T05:13:29,753 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=78, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure a12ea80f78b339cfbfc0e18b5338df0c, server=d1ba675678c9,39291,1731042803020}] 2024-11-08T05:13:29,757 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7830dc6b1acea6e1d468f8008d8a82b9, server=d1ba675678c9,39291,1731042803020}] 2024-11-08T05:13:29,790 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(132): Open test,zzz,1731042805591.e61dc9e059921fddc5ad7f6f7971acc6. 2024-11-08T05:13:29,790 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7752): Opening region: {ENCODED => e61dc9e059921fddc5ad7f6f7971acc6, NAME => 'test,zzz,1731042805591.e61dc9e059921fddc5ad7f6f7971acc6.', STARTKEY => 'zzz', ENDKEY => ''} 2024-11-08T05:13:29,790 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test e61dc9e059921fddc5ad7f6f7971acc6 2024-11-08T05:13:29,790 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(898): Instantiated test,zzz,1731042805591.e61dc9e059921fddc5ad7f6f7971acc6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T05:13:29,790 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7794): checking encryption for e61dc9e059921fddc5ad7f6f7971acc6 2024-11-08T05:13:29,790 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7797): checking classloading for e61dc9e059921fddc5ad7f6f7971acc6 2024-11-08T05:13:29,793 INFO [StoreOpener-e61dc9e059921fddc5ad7f6f7971acc6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region e61dc9e059921fddc5ad7f6f7971acc6 2024-11-08T05:13:29,795 INFO [StoreOpener-e61dc9e059921fddc5ad7f6f7971acc6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e61dc9e059921fddc5ad7f6f7971acc6 columnFamilyName family 2024-11-08T05:13:29,795 DEBUG [StoreOpener-e61dc9e059921fddc5ad7f6f7971acc6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T05:13:29,795 INFO [StoreOpener-e61dc9e059921fddc5ad7f6f7971acc6-1 {}] regionserver.HStore(327): Store=e61dc9e059921fddc5ad7f6f7971acc6/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T05:13:29,796 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1038): replaying wal for e61dc9e059921fddc5ad7f6f7971acc6 2024-11-08T05:13:29,797 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/default/test/e61dc9e059921fddc5ad7f6f7971acc6 2024-11-08T05:13:29,798 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/default/test/e61dc9e059921fddc5ad7f6f7971acc6 2024-11-08T05:13:29,799 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1048): stopping wal replay for e61dc9e059921fddc5ad7f6f7971acc6 2024-11-08T05:13:29,799 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1060): Cleaning up temporary data for e61dc9e059921fddc5ad7f6f7971acc6 2024-11-08T05:13:29,801 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1093): writing seq id for e61dc9e059921fddc5ad7f6f7971acc6 2024-11-08T05:13:29,802 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1114): Opened e61dc9e059921fddc5ad7f6f7971acc6; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68696869, jitterRate=0.023663118481636047}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T05:13:29,802 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e61dc9e059921fddc5ad7f6f7971acc6 2024-11-08T05:13:29,803 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1006): Region open journal for e61dc9e059921fddc5ad7f6f7971acc6: Running coprocessor pre-open hook at 1731042809792Writing region info on filesystem at 1731042809792Initializing all the Stores at 1731042809793 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731042809793Cleaning up temporary data from old regions at 1731042809799 (+6 ms)Running coprocessor post-open hooks at 1731042809802 (+3 ms)Region opened successfully at 1731042809802 2024-11-08T05:13:29,803 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2236): Post open deploy tasks for test,zzz,1731042805591.e61dc9e059921fddc5ad7f6f7971acc6., pid=77, masterSystemTime=1731042809785 2024-11-08T05:13:29,806 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2266): Finished post open deploy task for test,zzz,1731042805591.e61dc9e059921fddc5ad7f6f7971acc6. 2024-11-08T05:13:29,806 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(153): Opened test,zzz,1731042805591.e61dc9e059921fddc5ad7f6f7971acc6. 2024-11-08T05:13:29,807 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] handler.AssignRegionHandler(132): Open test,nnn,1731042805591.868527ec322eba1d7cbd6371701f7ad2. 2024-11-08T05:13:29,807 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(7752): Opening region: {ENCODED => 868527ec322eba1d7cbd6371701f7ad2, NAME => 'test,nnn,1731042805591.868527ec322eba1d7cbd6371701f7ad2.', STARTKEY => 'nnn', ENDKEY => 'ooo'} 2024-11-08T05:13:29,807 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 868527ec322eba1d7cbd6371701f7ad2 2024-11-08T05:13:29,807 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(898): Instantiated test,nnn,1731042805591.868527ec322eba1d7cbd6371701f7ad2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T05:13:29,807 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(7794): checking encryption for 868527ec322eba1d7cbd6371701f7ad2 2024-11-08T05:13:29,807 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(7797): checking classloading for 868527ec322eba1d7cbd6371701f7ad2 2024-11-08T05:13:29,809 WARN [PEWorker-1 {}] procedure2.ProcedureTestingUtility(201): Set Kill before store update in rollback to: true 2024-11-08T05:13:29,809 INFO [StoreOpener-868527ec322eba1d7cbd6371701f7ad2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 868527ec322eba1d7cbd6371701f7ad2 2024-11-08T05:13:29,809 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=77, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure e61dc9e059921fddc5ad7f6f7971acc6, server=d1ba675678c9,41485,1731042802970 2024-11-08T05:13:29,810 INFO [StoreOpener-868527ec322eba1d7cbd6371701f7ad2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 868527ec322eba1d7cbd6371701f7ad2 columnFamilyName family 2024-11-08T05:13:29,810 DEBUG [StoreOpener-868527ec322eba1d7cbd6371701f7ad2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T05:13:29,810 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1844): CODE-BUG: Uncaught runtime exception: pid=77, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure e61dc9e059921fddc5ad7f6f7971acc6, server=d1ba675678c9,41485,1731042802970 java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T05:13:29,811 INFO [StoreOpener-868527ec322eba1d7cbd6371701f7ad2-1 {}] regionserver.HStore(327): Store=868527ec322eba1d7cbd6371701f7ad2/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T05:13:29,811 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1038): replaying wal for 868527ec322eba1d7cbd6371701f7ad2 2024-11-08T05:13:29,813 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/default/test/868527ec322eba1d7cbd6371701f7ad2 2024-11-08T05:13:29,814 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/default/test/868527ec322eba1d7cbd6371701f7ad2 2024-11-08T05:13:29,815 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1048): stopping wal replay for 868527ec322eba1d7cbd6371701f7ad2 2024-11-08T05:13:29,815 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1060): Cleaning up temporary data for 868527ec322eba1d7cbd6371701f7ad2 2024-11-08T05:13:29,817 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1093): writing seq id for 868527ec322eba1d7cbd6371701f7ad2 2024-11-08T05:13:29,819 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1114): Opened 868527ec322eba1d7cbd6371701f7ad2; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65697193, jitterRate=-0.021035537123680115}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T05:13:29,819 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 868527ec322eba1d7cbd6371701f7ad2 2024-11-08T05:13:29,819 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1006): Region open journal for 868527ec322eba1d7cbd6371701f7ad2: Running coprocessor pre-open hook at 1731042809807Writing region info on filesystem at 1731042809808 (+1 ms)Initializing all the Stores at 1731042809809 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731042809809Cleaning up temporary data from old regions at 1731042809815 (+6 ms)Running coprocessor post-open hooks at 1731042809819 (+4 ms)Region opened successfully at 1731042809819 2024-11-08T05:13:29,820 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegionServer(2236): Post open deploy tasks for test,nnn,1731042805591.868527ec322eba1d7cbd6371701f7ad2., pid=72, masterSystemTime=1731042809785 2024-11-08T05:13:29,823 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegionServer(2266): Finished post open deploy task for test,nnn,1731042805591.868527ec322eba1d7cbd6371701f7ad2. 2024-11-08T05:13:29,821 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=true, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure e61dc9e059921fddc5ad7f6f7971acc6, server=d1ba675678c9,41485,1731042802970:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d1ba675678c9,46753,1731042802851, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1845) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 4 more 2024-11-08T05:13:29,823 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] handler.AssignRegionHandler(153): Opened test,nnn,1731042805591.868527ec322eba1d7cbd6371701f7ad2. 2024-11-08T05:13:29,824 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] handler.AssignRegionHandler(132): Open test,ggg,1731042805591.9b3274d80dcb7deed617c19ac1de4a68. 2024-11-08T05:13:29,824 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(7752): Opening region: {ENCODED => 9b3274d80dcb7deed617c19ac1de4a68, NAME => 'test,ggg,1731042805591.9b3274d80dcb7deed617c19ac1de4a68.', STARTKEY => 'ggg', ENDKEY => 'hhh'} 2024-11-08T05:13:29,824 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-08T05:13:29,824 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=79, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7830dc6b1acea6e1d468f8008d8a82b9, server=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:29,824 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T05:13:29,824 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 9b3274d80dcb7deed617c19ac1de4a68 2024-11-08T05:13:29,824 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(898): Instantiated test,ggg,1731042805591.9b3274d80dcb7deed617c19ac1de4a68.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T05:13:29,825 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(7794): checking encryption for 9b3274d80dcb7deed617c19ac1de4a68 2024-11-08T05:13:29,825 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(7797): checking classloading for 9b3274d80dcb7deed617c19ac1de4a68 2024-11-08T05:13:29,825 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=79, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7830dc6b1acea6e1d468f8008d8a82b9, server=d1ba675678c9,39291,1731042803020 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T05:13:29,826 INFO [StoreOpener-9b3274d80dcb7deed617c19ac1de4a68-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 9b3274d80dcb7deed617c19ac1de4a68 2024-11-08T05:13:29,827 INFO [StoreOpener-9b3274d80dcb7deed617c19ac1de4a68-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9b3274d80dcb7deed617c19ac1de4a68 columnFamilyName family 2024-11-08T05:13:29,828 DEBUG [StoreOpener-9b3274d80dcb7deed617c19ac1de4a68-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T05:13:29,828 INFO [StoreOpener-9b3274d80dcb7deed617c19ac1de4a68-1 {}] regionserver.HStore(327): Store=9b3274d80dcb7deed617c19ac1de4a68/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T05:13:29,829 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1038): replaying wal for 9b3274d80dcb7deed617c19ac1de4a68 2024-11-08T05:13:29,830 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/default/test/9b3274d80dcb7deed617c19ac1de4a68 2024-11-08T05:13:29,832 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/default/test/9b3274d80dcb7deed617c19ac1de4a68 2024-11-08T05:13:29,832 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1048): stopping wal replay for 9b3274d80dcb7deed617c19ac1de4a68 2024-11-08T05:13:29,832 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1060): Cleaning up temporary data for 9b3274d80dcb7deed617c19ac1de4a68 2024-11-08T05:13:29,835 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1093): writing seq id for 9b3274d80dcb7deed617c19ac1de4a68 2024-11-08T05:13:29,837 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1114): Opened 9b3274d80dcb7deed617c19ac1de4a68; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73564027, jitterRate=0.09618942439556122}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T05:13:29,837 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9b3274d80dcb7deed617c19ac1de4a68 2024-11-08T05:13:29,838 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1006): Region open journal for 9b3274d80dcb7deed617c19ac1de4a68: Running coprocessor pre-open hook at 1731042809825Writing region info on filesystem at 1731042809825Initializing all the Stores at 1731042809826 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731042809826Cleaning up temporary data from old regions at 1731042809832 (+6 ms)Running coprocessor post-open hooks at 1731042809837 (+5 ms)Region opened successfully at 1731042809837 2024-11-08T05:13:29,839 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ggg,1731042805591.9b3274d80dcb7deed617c19ac1de4a68., pid=73, masterSystemTime=1731042809785 2024-11-08T05:13:29,847 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ggg,1731042805591.9b3274d80dcb7deed617c19ac1de4a68. 2024-11-08T05:13:29,848 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] handler.AssignRegionHandler(153): Opened test,ggg,1731042805591.9b3274d80dcb7deed617c19ac1de4a68. 2024-11-08T05:13:29,848 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] handler.AssignRegionHandler(132): Open test,ttt,1731042805591.445b14856951586918c91d4ba342f1ff. 2024-11-08T05:13:29,848 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7752): Opening region: {ENCODED => 445b14856951586918c91d4ba342f1ff, NAME => 'test,ttt,1731042805591.445b14856951586918c91d4ba342f1ff.', STARTKEY => 'ttt', ENDKEY => 'uuu'} 2024-11-08T05:13:29,849 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 445b14856951586918c91d4ba342f1ff 2024-11-08T05:13:29,849 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(898): Instantiated test,ttt,1731042805591.445b14856951586918c91d4ba342f1ff.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T05:13:29,849 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7794): checking encryption for 445b14856951586918c91d4ba342f1ff 2024-11-08T05:13:29,849 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7797): checking classloading for 445b14856951586918c91d4ba342f1ff 2024-11-08T05:13:29,851 INFO [StoreOpener-445b14856951586918c91d4ba342f1ff-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 445b14856951586918c91d4ba342f1ff 2024-11-08T05:13:29,852 INFO [StoreOpener-445b14856951586918c91d4ba342f1ff-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 445b14856951586918c91d4ba342f1ff columnFamilyName family 2024-11-08T05:13:29,852 DEBUG [StoreOpener-445b14856951586918c91d4ba342f1ff-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T05:13:29,852 INFO [StoreOpener-445b14856951586918c91d4ba342f1ff-1 {}] regionserver.HStore(327): Store=445b14856951586918c91d4ba342f1ff/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T05:13:29,853 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1038): replaying wal for 445b14856951586918c91d4ba342f1ff 2024-11-08T05:13:29,853 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/default/test/445b14856951586918c91d4ba342f1ff 2024-11-08T05:13:29,855 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/default/test/445b14856951586918c91d4ba342f1ff 2024-11-08T05:13:29,855 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1048): stopping wal replay for 445b14856951586918c91d4ba342f1ff 2024-11-08T05:13:29,855 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1060): Cleaning up temporary data for 445b14856951586918c91d4ba342f1ff 2024-11-08T05:13:29,858 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1093): writing seq id for 445b14856951586918c91d4ba342f1ff 2024-11-08T05:13:29,859 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1114): Opened 445b14856951586918c91d4ba342f1ff; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73271149, jitterRate=0.0918252021074295}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T05:13:29,859 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 445b14856951586918c91d4ba342f1ff 2024-11-08T05:13:29,859 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1006): Region open journal for 445b14856951586918c91d4ba342f1ff: Running coprocessor pre-open hook at 1731042809849Writing region info on filesystem at 1731042809849Initializing all the Stores at 1731042809850 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731042809850Cleaning up temporary data from old regions at 1731042809855 (+5 ms)Running coprocessor post-open hooks at 1731042809859 (+4 ms)Region opened successfully at 1731042809859 2024-11-08T05:13:29,860 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ttt,1731042805591.445b14856951586918c91d4ba342f1ff., pid=76, masterSystemTime=1731042809785 2024-11-08T05:13:29,862 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ttt,1731042805591.445b14856951586918c91d4ba342f1ff. 2024-11-08T05:13:29,862 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] handler.AssignRegionHandler(153): Opened test,ttt,1731042805591.445b14856951586918c91d4ba342f1ff. 2024-11-08T05:13:29,887 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-08T05:13:29,888 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T05:13:29,888 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-08T05:13:29,888 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-08T05:13:29,889 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-08T05:13:29,889 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-08T05:13:29,889 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-08T05:13:29,890 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-08T05:13:29,891 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-08T05:13:29,893 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37587 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T05:13:29,898 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(132): Open test,ddd,1731042805591.64ddf686fed30cdb23b2e874efec598d. 2024-11-08T05:13:29,898 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7752): Opening region: {ENCODED => 64ddf686fed30cdb23b2e874efec598d, NAME => 'test,ddd,1731042805591.64ddf686fed30cdb23b2e874efec598d.', STARTKEY => 'ddd', ENDKEY => 'eee'} 2024-11-08T05:13:29,899 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 64ddf686fed30cdb23b2e874efec598d 2024-11-08T05:13:29,899 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(898): Instantiated test,ddd,1731042805591.64ddf686fed30cdb23b2e874efec598d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T05:13:29,899 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7794): checking encryption for 64ddf686fed30cdb23b2e874efec598d 2024-11-08T05:13:29,899 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7797): checking classloading for 64ddf686fed30cdb23b2e874efec598d 2024-11-08T05:13:29,900 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-08T05:13:29,900 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37587 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T05:13:29,901 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T05:13:29,901 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=64, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T05:13:29,901 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T05:13:29,901 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T05:13:29,901 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T05:13:29,901 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T05:13:29,901 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T05:13:29,901 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T05:13:29,901 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T05:13:29,901 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T05:13:29,901 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T05:13:29,901 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T05:13:29,901 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T05:13:29,901 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T05:13:29,901 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T05:13:29,901 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T05:13:29,901 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T05:13:29,901 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T05:13:29,901 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T05:13:29,901 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T05:13:29,901 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T05:13:29,901 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T05:13:29,902 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-08T05:13:29,903 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-08T05:13:29,904 INFO [StoreOpener-64ddf686fed30cdb23b2e874efec598d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 64ddf686fed30cdb23b2e874efec598d 2024-11-08T05:13:29,906 INFO [StoreOpener-64ddf686fed30cdb23b2e874efec598d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 64ddf686fed30cdb23b2e874efec598d columnFamilyName family 2024-11-08T05:13:29,906 DEBUG [StoreOpener-64ddf686fed30cdb23b2e874efec598d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T05:13:29,907 INFO [StoreOpener-64ddf686fed30cdb23b2e874efec598d-1 {}] regionserver.HStore(327): Store=64ddf686fed30cdb23b2e874efec598d/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T05:13:29,907 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1038): replaying wal for 64ddf686fed30cdb23b2e874efec598d 2024-11-08T05:13:29,909 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/default/test/64ddf686fed30cdb23b2e874efec598d 2024-11-08T05:13:29,910 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure e61dc9e059921fddc5ad7f6f7971acc6, server=d1ba675678c9,41485,1731042802970:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d1ba675678c9,46753,1731042802851, splitWal=true, meta=true 2024-11-08T05:13:29,910 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure d1ba675678c9%2C46753%2C1731042802851.meta.1731042805109.meta, worker=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:29,910 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d1ba675678c9%2C46753%2C1731042802851.meta.1731042805109.meta, worker=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:29,910 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T05:13:29,910 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d1ba675678c9,41485,1731042802970 2024-11-08T05:13:29,910 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure d1ba675678c9%2C46753%2C1731042802851.1731042804603, worker=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:29,910 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d1ba675678c9%2C46753%2C1731042802851.1731042804603, worker=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:29,910 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e61dc9e059921fddc5ad7f6f7971acc6, ASSIGN 2024-11-08T05:13:29,910 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=81806cb184c70c9dc0df408694092d85, ASSIGN 2024-11-08T05:13:29,910 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=868527ec322eba1d7cbd6371701f7ad2, ASSIGN 2024-11-08T05:13:29,910 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a12ea80f78b339cfbfc0e18b5338df0c, ASSIGN 2024-11-08T05:13:29,911 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=445b14856951586918c91d4ba342f1ff, ASSIGN 2024-11-08T05:13:29,911 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9b3274d80dcb7deed617c19ac1de4a68, ASSIGN 2024-11-08T05:13:29,911 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7830dc6b1acea6e1d468f8008d8a82b9, ASSIGN 2024-11-08T05:13:29,911 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=64ddf686fed30cdb23b2e874efec598d, ASSIGN 2024-11-08T05:13:29,911 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 868527ec322eba1d7cbd6371701f7ad2, server=d1ba675678c9,41485,1731042802970 2024-11-08T05:13:29,911 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9b3274d80dcb7deed617c19ac1de4a68, server=d1ba675678c9,41485,1731042802970 2024-11-08T05:13:29,911 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 81806cb184c70c9dc0df408694092d85, server=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:29,911 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 64ddf686fed30cdb23b2e874efec598d, server=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:29,911 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 445b14856951586918c91d4ba342f1ff, server=d1ba675678c9,41485,1731042802970 2024-11-08T05:13:29,911 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=64, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure e61dc9e059921fddc5ad7f6f7971acc6, server=d1ba675678c9,41485,1731042802970:java.lang.RuntimeException: inject code bug; OpenRegionProcedure e61dc9e059921fddc5ad7f6f7971acc6, server=d1ba675678c9,41485,1731042802970 2024-11-08T05:13:29,911 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/default/test/64ddf686fed30cdb23b2e874efec598d 2024-11-08T05:13:29,911 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure a12ea80f78b339cfbfc0e18b5338df0c, server=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:29,911 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7830dc6b1acea6e1d468f8008d8a82b9, server=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:29,911 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure e61dc9e059921fddc5ad7f6f7971acc6, server=d1ba675678c9,41485,1731042802970:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d1ba675678c9,46753,1731042802851, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:29,911 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure d1ba675678c9%2C46753%2C1731042802851.meta.1731042805109.meta, worker=d1ba675678c9,39291,1731042803020 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:29,911 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d1ba675678c9%2C46753%2C1731042802851.meta.1731042805109.meta, worker=d1ba675678c9,39291,1731042803020 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:29,912 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:29,912 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1048): stopping wal replay for 64ddf686fed30cdb23b2e874efec598d 2024-11-08T05:13:29,912 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d1ba675678c9,41485,1731042802970 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:29,912 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1060): Cleaning up temporary data for 64ddf686fed30cdb23b2e874efec598d 2024-11-08T05:13:29,912 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure d1ba675678c9%2C46753%2C1731042802851.1731042804603, worker=d1ba675678c9,39291,1731042803020 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:29,912 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d1ba675678c9%2C46753%2C1731042802851.1731042804603, worker=d1ba675678c9,39291,1731042803020 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:29,912 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e61dc9e059921fddc5ad7f6f7971acc6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:29,912 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e61dc9e059921fddc5ad7f6f7971acc6, ASSIGN 2024-11-08T05:13:29,912 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=81806cb184c70c9dc0df408694092d85, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:29,912 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=81806cb184c70c9dc0df408694092d85, ASSIGN 2024-11-08T05:13:29,912 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=868527ec322eba1d7cbd6371701f7ad2, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:29,912 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=868527ec322eba1d7cbd6371701f7ad2, ASSIGN 2024-11-08T05:13:29,912 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a12ea80f78b339cfbfc0e18b5338df0c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:29,912 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a12ea80f78b339cfbfc0e18b5338df0c, ASSIGN 2024-11-08T05:13:29,912 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=445b14856951586918c91d4ba342f1ff, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:29,912 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=445b14856951586918c91d4ba342f1ff, ASSIGN 2024-11-08T05:13:29,912 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9b3274d80dcb7deed617c19ac1de4a68, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:29,912 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9b3274d80dcb7deed617c19ac1de4a68, ASSIGN 2024-11-08T05:13:29,912 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7830dc6b1acea6e1d468f8008d8a82b9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:29,912 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7830dc6b1acea6e1d468f8008d8a82b9, ASSIGN 2024-11-08T05:13:29,913 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=64ddf686fed30cdb23b2e874efec598d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:29,913 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=64ddf686fed30cdb23b2e874efec598d, ASSIGN 2024-11-08T05:13:29,913 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 868527ec322eba1d7cbd6371701f7ad2, server=d1ba675678c9,41485,1731042802970 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:29,913 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9b3274d80dcb7deed617c19ac1de4a68, server=d1ba675678c9,41485,1731042802970 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:29,913 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 81806cb184c70c9dc0df408694092d85, server=d1ba675678c9,39291,1731042803020 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:29,913 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 64ddf686fed30cdb23b2e874efec598d, server=d1ba675678c9,39291,1731042803020 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:29,913 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 445b14856951586918c91d4ba342f1ff, server=d1ba675678c9,41485,1731042802970 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:29,913 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=64, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure e61dc9e059921fddc5ad7f6f7971acc6, server=d1ba675678c9,41485,1731042802970:java.lang.RuntimeException: inject code bug; OpenRegionProcedure e61dc9e059921fddc5ad7f6f7971acc6, server=d1ba675678c9,41485,1731042802970 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:29,913 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure a12ea80f78b339cfbfc0e18b5338df0c, server=d1ba675678c9,39291,1731042803020 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:29,913 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7830dc6b1acea6e1d468f8008d8a82b9, server=d1ba675678c9,39291,1731042803020 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:29,914 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 22 msec 2024-11-08T05:13:29,915 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1093): writing seq id for 64ddf686fed30cdb23b2e874efec598d 2024-11-08T05:13:29,916 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1114): Opened 64ddf686fed30cdb23b2e874efec598d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75429636, jitterRate=0.12398916482925415}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T05:13:29,916 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 64ddf686fed30cdb23b2e874efec598d 2024-11-08T05:13:29,916 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1006): Region open journal for 64ddf686fed30cdb23b2e874efec598d: Running coprocessor pre-open hook at 1731042809899Writing region info on filesystem at 1731042809899Initializing all the Stores at 1731042809901 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731042809901Cleaning up temporary data from old regions at 1731042809912 (+11 ms)Running coprocessor post-open hooks at 1731042809916 (+4 ms)Region opened successfully at 1731042809916 2024-11-08T05:13:29,917 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ddd,1731042805591.64ddf686fed30cdb23b2e874efec598d., pid=75, masterSystemTime=1731042809893 2024-11-08T05:13:29,918 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=61 2024-11-08T05:13:29,918 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=d1ba675678c9,41485,1731042802970, table=hbase:meta, region=1588230740 2024-11-08T05:13:29,919 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d1ba675678c9,41485,1731042802970, state=OPEN 2024-11-08T05:13:29,918 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37587 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T05:13:29,921 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2367): Failed report transition server { host_name: "d1ba675678c9" port: 39291 start_code: 1731042803020 } transition { transition_code: OPENED region_info { region_id: 1731042805591 table_name { namespace: "default" qualifier: "test" } start_key: "ddd" end_key: "eee" offline: false split: false replica_id: 0 } open_seq_num: 2 proc_id: 75 initiating_master_active_time: 1731042803126 }; retry (#0) after 1002ms delay (Master is coming online...). org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:403) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.getRemoteException(ProtobufUtil.java:381) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportRegionStateTransition(HRegionServer.java:2358) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2257) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-08T05:13:29,966 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39291-0x10118dcefab0003, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T05:13:29,966 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41485-0x10118dcefab0002, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T05:13:29,966 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37587-0x10118dcefab0000, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T05:13:29,966 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T05:13:29,966 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T05:13:29,966 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T05:13:29,968 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e61dc9e059921fddc5ad7f6f7971acc6, ASSIGN to state=OFFLINE, location=null, table=test, region=e61dc9e059921fddc5ad7f6f7971acc6 to restore RIT 2024-11-08T05:13:29,968 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=81806cb184c70c9dc0df408694092d85, ASSIGN to state=OFFLINE, location=null, table=test, region=81806cb184c70c9dc0df408694092d85 to restore RIT 2024-11-08T05:13:29,968 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=868527ec322eba1d7cbd6371701f7ad2, ASSIGN to state=OFFLINE, location=null, table=test, region=868527ec322eba1d7cbd6371701f7ad2 to restore RIT 2024-11-08T05:13:29,968 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a12ea80f78b339cfbfc0e18b5338df0c, ASSIGN to state=OFFLINE, location=null, table=test, region=a12ea80f78b339cfbfc0e18b5338df0c to restore RIT 2024-11-08T05:13:29,968 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=445b14856951586918c91d4ba342f1ff, ASSIGN to state=OFFLINE, location=null, table=test, region=445b14856951586918c91d4ba342f1ff to restore RIT 2024-11-08T05:13:29,968 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9b3274d80dcb7deed617c19ac1de4a68, ASSIGN to state=OFFLINE, location=null, table=test, region=9b3274d80dcb7deed617c19ac1de4a68 to restore RIT 2024-11-08T05:13:29,968 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7830dc6b1acea6e1d468f8008d8a82b9, ASSIGN to state=OFFLINE, location=null, table=test, region=7830dc6b1acea6e1d468f8008d8a82b9 to restore RIT 2024-11-08T05:13:29,968 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=64ddf686fed30cdb23b2e874efec598d, ASSIGN to state=OFFLINE, location=null, table=test, region=64ddf686fed30cdb23b2e874efec598d to restore RIT 2024-11-08T05:13:29,969 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731042839969 2024-11-08T05:13:29,969 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-08T05:13:29,970 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure e61dc9e059921fddc5ad7f6f7971acc6, server=d1ba675678c9,41485,1731042802970:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d1ba675678c9,46753,1731042802851, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T05:13:29,972 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-08T05:13:29,974 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-08T05:13:29,974 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=78, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure a12ea80f78b339cfbfc0e18b5338df0c, server=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:29,974 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T05:13:29,974 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=78, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure a12ea80f78b339cfbfc0e18b5338df0c, server=d1ba675678c9,39291,1731042803020 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T05:13:30,000 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6240e38715889f3bb77bb89478c81a6c, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,000 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eeaad0e7e7aa66f0b51c273c0aaf953c, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,001 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=68bbd0353f97ee12228f7798b0b987ea, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,001 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=64ddf686fed30cdb23b2e874efec598d, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,001 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d7680a9929f15640ca61551f186f33c1, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,001 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=65d5312dbd2f96bf3b16bdfd7504dff6, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,002 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9b3274d80dcb7deed617c19ac1de4a68, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,002 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2d23e3e11ecc480e314e989d1bb8d0c3, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,002 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fe1545b16a81758dcb9430dbf98c4ab9, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,003 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a12ea80f78b339cfbfc0e18b5338df0c, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,003 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e1c63297d3a8aea2fdbd1842301c811, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,003 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=40440b5f39bb5421276aa0b156481a9e, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,003 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=27c9aaf34aaf98cd5bc8030630ae7fe3, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,004 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=868527ec322eba1d7cbd6371701f7ad2, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,004 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d951febea78018bb9f10f07d8788575d, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,004 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b0ebd9a9794acc56e430b36d1f17621c, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,005 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7830dc6b1acea6e1d468f8008d8a82b9, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,005 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12f1148a7b8db3e4f515337b8b7746fa, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,005 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e26dcb2339b364f3699d110c15cd4bb3, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,005 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=445b14856951586918c91d4ba342f1ff, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,006 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=14efcc1b9822cc74f4d4a96648555167, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,006 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7c8f72849b2e1719ed903afc73e0571a, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,006 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=81806cb184c70c9dc0df408694092d85, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,006 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=87b3455cffd1f4e06e5901b3bcbed9dc, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,007 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=de358f808110a48fa32e0533fefdc95c, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,007 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e61dc9e059921fddc5ad7f6f7971acc6, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,007 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-08T05:13:30,007 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731042870007 2024-11-08T05:13:30,007 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731042930007 2024-11-08T05:13:30,007 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 38 msec 2024-11-08T05:13:30,007 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-08T05:13:30,007 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T05:13:30,007 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-08T05:13:30,008 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-08T05:13:30,008 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-08T05:13:30,008 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-08T05:13:30,008 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-08T05:13:30,008 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-08T05:13:30,009 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-08T05:13:30,015 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-08T05:13:30,015 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T05:13:30,015 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=64, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T05:13:30,015 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T05:13:30,015 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T05:13:30,015 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T05:13:30,015 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T05:13:30,015 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T05:13:30,015 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T05:13:30,015 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T05:13:30,016 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T05:13:30,016 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T05:13:30,016 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T05:13:30,016 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T05:13:30,016 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T05:13:30,016 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T05:13:30,016 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T05:13:30,016 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T05:13:30,016 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T05:13:30,016 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T05:13:30,016 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T05:13:30,016 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T05:13:30,016 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-08T05:13:30,018 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-08T05:13:30,022 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure e61dc9e059921fddc5ad7f6f7971acc6, server=d1ba675678c9,41485,1731042802970:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d1ba675678c9,46753,1731042802851, splitWal=true, meta=true 2024-11-08T05:13:30,022 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure d1ba675678c9%2C46753%2C1731042802851.meta.1731042805109.meta, worker=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:30,022 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d1ba675678c9%2C46753%2C1731042802851.meta.1731042805109.meta, worker=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:30,022 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T05:13:30,022 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d1ba675678c9,41485,1731042802970 2024-11-08T05:13:30,022 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure d1ba675678c9%2C46753%2C1731042802851.1731042804603, worker=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:30,022 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d1ba675678c9%2C46753%2C1731042802851.1731042804603, worker=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:30,022 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e61dc9e059921fddc5ad7f6f7971acc6, ASSIGN 2024-11-08T05:13:30,022 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=81806cb184c70c9dc0df408694092d85, ASSIGN 2024-11-08T05:13:30,022 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=868527ec322eba1d7cbd6371701f7ad2, ASSIGN 2024-11-08T05:13:30,022 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a12ea80f78b339cfbfc0e18b5338df0c, ASSIGN 2024-11-08T05:13:30,022 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=445b14856951586918c91d4ba342f1ff, ASSIGN 2024-11-08T05:13:30,022 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9b3274d80dcb7deed617c19ac1de4a68, ASSIGN 2024-11-08T05:13:30,022 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7830dc6b1acea6e1d468f8008d8a82b9, ASSIGN 2024-11-08T05:13:30,023 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=64ddf686fed30cdb23b2e874efec598d, ASSIGN 2024-11-08T05:13:30,023 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 868527ec322eba1d7cbd6371701f7ad2, server=d1ba675678c9,41485,1731042802970 2024-11-08T05:13:30,023 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9b3274d80dcb7deed617c19ac1de4a68, server=d1ba675678c9,41485,1731042802970 2024-11-08T05:13:30,023 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 81806cb184c70c9dc0df408694092d85, server=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:30,023 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 64ddf686fed30cdb23b2e874efec598d, server=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:30,023 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 445b14856951586918c91d4ba342f1ff, server=d1ba675678c9,41485,1731042802970 2024-11-08T05:13:30,023 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=64, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure e61dc9e059921fddc5ad7f6f7971acc6, server=d1ba675678c9,41485,1731042802970:java.lang.RuntimeException: inject code bug; OpenRegionProcedure e61dc9e059921fddc5ad7f6f7971acc6, server=d1ba675678c9,41485,1731042802970 2024-11-08T05:13:30,023 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure a12ea80f78b339cfbfc0e18b5338df0c, server=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:30,023 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure e61dc9e059921fddc5ad7f6f7971acc6, server=d1ba675678c9,41485,1731042802970:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d1ba675678c9,46753,1731042802851, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:30,023 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure d1ba675678c9%2C46753%2C1731042802851.meta.1731042805109.meta, worker=d1ba675678c9,39291,1731042803020 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,023 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d1ba675678c9%2C46753%2C1731042802851.meta.1731042805109.meta, worker=d1ba675678c9,39291,1731042803020 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,023 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,023 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d1ba675678c9,41485,1731042802970 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,023 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure d1ba675678c9%2C46753%2C1731042802851.1731042804603, worker=d1ba675678c9,39291,1731042803020 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,023 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d1ba675678c9%2C46753%2C1731042802851.1731042804603, worker=d1ba675678c9,39291,1731042803020 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,023 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e61dc9e059921fddc5ad7f6f7971acc6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:30,023 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e61dc9e059921fddc5ad7f6f7971acc6, ASSIGN 2024-11-08T05:13:30,024 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=81806cb184c70c9dc0df408694092d85, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:30,024 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=81806cb184c70c9dc0df408694092d85, ASSIGN 2024-11-08T05:13:30,024 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=868527ec322eba1d7cbd6371701f7ad2, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:30,024 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=868527ec322eba1d7cbd6371701f7ad2, ASSIGN 2024-11-08T05:13:30,024 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a12ea80f78b339cfbfc0e18b5338df0c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:30,024 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a12ea80f78b339cfbfc0e18b5338df0c, ASSIGN 2024-11-08T05:13:30,024 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=445b14856951586918c91d4ba342f1ff, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:30,024 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=445b14856951586918c91d4ba342f1ff, ASSIGN 2024-11-08T05:13:30,024 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9b3274d80dcb7deed617c19ac1de4a68, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:30,024 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9b3274d80dcb7deed617c19ac1de4a68, ASSIGN 2024-11-08T05:13:30,024 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7830dc6b1acea6e1d468f8008d8a82b9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:30,024 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7830dc6b1acea6e1d468f8008d8a82b9, ASSIGN 2024-11-08T05:13:30,024 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=64ddf686fed30cdb23b2e874efec598d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:30,024 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=64ddf686fed30cdb23b2e874efec598d, ASSIGN 2024-11-08T05:13:30,024 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 868527ec322eba1d7cbd6371701f7ad2, server=d1ba675678c9,41485,1731042802970 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,024 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9b3274d80dcb7deed617c19ac1de4a68, server=d1ba675678c9,41485,1731042802970 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,024 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 81806cb184c70c9dc0df408694092d85, server=d1ba675678c9,39291,1731042803020 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,024 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 64ddf686fed30cdb23b2e874efec598d, server=d1ba675678c9,39291,1731042803020 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,025 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 445b14856951586918c91d4ba342f1ff, server=d1ba675678c9,41485,1731042802970 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,025 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=64, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure e61dc9e059921fddc5ad7f6f7971acc6, server=d1ba675678c9,41485,1731042802970:java.lang.RuntimeException: inject code bug; OpenRegionProcedure e61dc9e059921fddc5ad7f6f7971acc6, server=d1ba675678c9,41485,1731042802970 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,025 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure a12ea80f78b339cfbfc0e18b5338df0c, server=d1ba675678c9,39291,1731042803020 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,025 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 15 msec 2024-11-08T05:13:30,026 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=61 2024-11-08T05:13:30,026 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=d1ba675678c9,41485,1731042802970, table=hbase:meta, region=1588230740 2024-11-08T05:13:30,026 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d1ba675678c9,41485,1731042802970, state=OPEN 2024-11-08T05:13:30,039 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41485-0x10118dcefab0002, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T05:13:30,039 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37587-0x10118dcefab0000, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T05:13:30,039 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39291-0x10118dcefab0003, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T05:13:30,039 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T05:13:30,039 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T05:13:30,039 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T05:13:30,040 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e61dc9e059921fddc5ad7f6f7971acc6, ASSIGN to state=OFFLINE, location=null, table=test, region=e61dc9e059921fddc5ad7f6f7971acc6 to restore RIT 2024-11-08T05:13:30,040 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=81806cb184c70c9dc0df408694092d85, ASSIGN to state=OFFLINE, location=null, table=test, region=81806cb184c70c9dc0df408694092d85 to restore RIT 2024-11-08T05:13:30,040 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=868527ec322eba1d7cbd6371701f7ad2, ASSIGN to state=OFFLINE, location=null, table=test, region=868527ec322eba1d7cbd6371701f7ad2 to restore RIT 2024-11-08T05:13:30,040 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a12ea80f78b339cfbfc0e18b5338df0c, ASSIGN to state=OFFLINE, location=null, table=test, region=a12ea80f78b339cfbfc0e18b5338df0c to restore RIT 2024-11-08T05:13:30,041 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=445b14856951586918c91d4ba342f1ff, ASSIGN to state=OFFLINE, location=null, table=test, region=445b14856951586918c91d4ba342f1ff to restore RIT 2024-11-08T05:13:30,041 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9b3274d80dcb7deed617c19ac1de4a68, ASSIGN to state=OFFLINE, location=null, table=test, region=9b3274d80dcb7deed617c19ac1de4a68 to restore RIT 2024-11-08T05:13:30,041 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7830dc6b1acea6e1d468f8008d8a82b9, ASSIGN to state=OFFLINE, location=null, table=test, region=7830dc6b1acea6e1d468f8008d8a82b9 to restore RIT 2024-11-08T05:13:30,041 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=64ddf686fed30cdb23b2e874efec598d, ASSIGN to state=OFFLINE, location=null, table=test, region=64ddf686fed30cdb23b2e874efec598d to restore RIT 2024-11-08T05:13:30,041 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731042840041 2024-11-08T05:13:30,041 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-08T05:13:30,041 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure e61dc9e059921fddc5ad7f6f7971acc6, server=d1ba675678c9,41485,1731042802970:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d1ba675678c9,46753,1731042802851, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T05:13:30,043 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-08T05:13:30,044 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-08T05:13:30,045 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=77, ppid=64, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure e61dc9e059921fddc5ad7f6f7971acc6, server=d1ba675678c9,41485,1731042802970:java.lang.RuntimeException: inject code bug; OpenRegionProcedure e61dc9e059921fddc5ad7f6f7971acc6, server=d1ba675678c9,41485,1731042802970 2024-11-08T05:13:30,045 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T05:13:30,045 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=77, ppid=64, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure e61dc9e059921fddc5ad7f6f7971acc6, server=d1ba675678c9,41485,1731042802970:java.lang.RuntimeException: inject code bug; OpenRegionProcedure e61dc9e059921fddc5ad7f6f7971acc6, server=d1ba675678c9,41485,1731042802970 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T05:13:30,050 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6240e38715889f3bb77bb89478c81a6c, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,050 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eeaad0e7e7aa66f0b51c273c0aaf953c, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,050 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=68bbd0353f97ee12228f7798b0b987ea, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,051 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=64ddf686fed30cdb23b2e874efec598d, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,051 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d7680a9929f15640ca61551f186f33c1, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,051 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=65d5312dbd2f96bf3b16bdfd7504dff6, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,051 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9b3274d80dcb7deed617c19ac1de4a68, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,051 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2d23e3e11ecc480e314e989d1bb8d0c3, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,052 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fe1545b16a81758dcb9430dbf98c4ab9, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,052 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a12ea80f78b339cfbfc0e18b5338df0c, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,052 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e1c63297d3a8aea2fdbd1842301c811, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,052 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=40440b5f39bb5421276aa0b156481a9e, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,052 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=27c9aaf34aaf98cd5bc8030630ae7fe3, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,052 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=868527ec322eba1d7cbd6371701f7ad2, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,053 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d951febea78018bb9f10f07d8788575d, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,053 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b0ebd9a9794acc56e430b36d1f17621c, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,053 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7830dc6b1acea6e1d468f8008d8a82b9, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,053 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12f1148a7b8db3e4f515337b8b7746fa, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,053 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e26dcb2339b364f3699d110c15cd4bb3, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,053 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=445b14856951586918c91d4ba342f1ff, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,054 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=14efcc1b9822cc74f4d4a96648555167, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,054 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7c8f72849b2e1719ed903afc73e0571a, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,054 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=81806cb184c70c9dc0df408694092d85, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,054 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=87b3455cffd1f4e06e5901b3bcbed9dc, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,054 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=de358f808110a48fa32e0533fefdc95c, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,054 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e61dc9e059921fddc5ad7f6f7971acc6, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,054 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-08T05:13:30,054 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731042870054 2024-11-08T05:13:30,054 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731042930054 2024-11-08T05:13:30,054 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 13 msec 2024-11-08T05:13:30,055 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-08T05:13:30,055 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T05:13:30,055 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-08T05:13:30,055 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-08T05:13:30,055 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-08T05:13:30,055 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-08T05:13:30,055 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-08T05:13:30,055 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-08T05:13:30,056 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-08T05:13:30,060 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-08T05:13:30,060 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T05:13:30,060 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=64, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T05:13:30,060 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T05:13:30,060 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T05:13:30,060 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T05:13:30,060 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T05:13:30,060 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T05:13:30,060 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T05:13:30,060 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T05:13:30,061 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T05:13:30,061 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T05:13:30,061 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T05:13:30,061 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T05:13:30,061 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T05:13:30,061 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T05:13:30,061 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T05:13:30,061 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T05:13:30,061 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T05:13:30,061 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T05:13:30,061 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T05:13:30,061 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-08T05:13:30,062 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-08T05:13:30,065 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure e61dc9e059921fddc5ad7f6f7971acc6, server=d1ba675678c9,41485,1731042802970:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d1ba675678c9,46753,1731042802851, splitWal=true, meta=true 2024-11-08T05:13:30,065 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure d1ba675678c9%2C46753%2C1731042802851.meta.1731042805109.meta, worker=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:30,065 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d1ba675678c9%2C46753%2C1731042802851.meta.1731042805109.meta, worker=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:30,065 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T05:13:30,065 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d1ba675678c9,41485,1731042802970 2024-11-08T05:13:30,065 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure d1ba675678c9%2C46753%2C1731042802851.1731042804603, worker=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:30,065 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d1ba675678c9%2C46753%2C1731042802851.1731042804603, worker=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:30,065 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e61dc9e059921fddc5ad7f6f7971acc6, ASSIGN 2024-11-08T05:13:30,065 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=81806cb184c70c9dc0df408694092d85, ASSIGN 2024-11-08T05:13:30,065 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=868527ec322eba1d7cbd6371701f7ad2, ASSIGN 2024-11-08T05:13:30,065 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a12ea80f78b339cfbfc0e18b5338df0c, ASSIGN 2024-11-08T05:13:30,065 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=445b14856951586918c91d4ba342f1ff, ASSIGN 2024-11-08T05:13:30,065 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9b3274d80dcb7deed617c19ac1de4a68, ASSIGN 2024-11-08T05:13:30,065 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7830dc6b1acea6e1d468f8008d8a82b9, ASSIGN 2024-11-08T05:13:30,065 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=64ddf686fed30cdb23b2e874efec598d, ASSIGN 2024-11-08T05:13:30,065 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 868527ec322eba1d7cbd6371701f7ad2, server=d1ba675678c9,41485,1731042802970 2024-11-08T05:13:30,065 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9b3274d80dcb7deed617c19ac1de4a68, server=d1ba675678c9,41485,1731042802970 2024-11-08T05:13:30,065 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 81806cb184c70c9dc0df408694092d85, server=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:30,066 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 64ddf686fed30cdb23b2e874efec598d, server=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:30,066 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 445b14856951586918c91d4ba342f1ff, server=d1ba675678c9,41485,1731042802970 2024-11-08T05:13:30,066 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=64, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure e61dc9e059921fddc5ad7f6f7971acc6, server=d1ba675678c9,41485,1731042802970:java.lang.RuntimeException: inject code bug; OpenRegionProcedure e61dc9e059921fddc5ad7f6f7971acc6, server=d1ba675678c9,41485,1731042802970 2024-11-08T05:13:30,066 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure e61dc9e059921fddc5ad7f6f7971acc6, server=d1ba675678c9,41485,1731042802970:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d1ba675678c9,46753,1731042802851, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:30,066 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure d1ba675678c9%2C46753%2C1731042802851.meta.1731042805109.meta, worker=d1ba675678c9,39291,1731042803020 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,066 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d1ba675678c9%2C46753%2C1731042802851.meta.1731042805109.meta, worker=d1ba675678c9,39291,1731042803020 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,066 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,066 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d1ba675678c9,41485,1731042802970 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,066 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure d1ba675678c9%2C46753%2C1731042802851.1731042804603, worker=d1ba675678c9,39291,1731042803020 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,066 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d1ba675678c9%2C46753%2C1731042802851.1731042804603, worker=d1ba675678c9,39291,1731042803020 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,066 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e61dc9e059921fddc5ad7f6f7971acc6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:30,066 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e61dc9e059921fddc5ad7f6f7971acc6, ASSIGN 2024-11-08T05:13:30,066 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=81806cb184c70c9dc0df408694092d85, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:30,066 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=81806cb184c70c9dc0df408694092d85, ASSIGN 2024-11-08T05:13:30,066 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=868527ec322eba1d7cbd6371701f7ad2, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:30,066 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=868527ec322eba1d7cbd6371701f7ad2, ASSIGN 2024-11-08T05:13:30,066 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a12ea80f78b339cfbfc0e18b5338df0c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:30,066 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a12ea80f78b339cfbfc0e18b5338df0c, ASSIGN 2024-11-08T05:13:30,066 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=445b14856951586918c91d4ba342f1ff, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:30,066 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=445b14856951586918c91d4ba342f1ff, ASSIGN 2024-11-08T05:13:30,066 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9b3274d80dcb7deed617c19ac1de4a68, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:30,066 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9b3274d80dcb7deed617c19ac1de4a68, ASSIGN 2024-11-08T05:13:30,067 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7830dc6b1acea6e1d468f8008d8a82b9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:30,067 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7830dc6b1acea6e1d468f8008d8a82b9, ASSIGN 2024-11-08T05:13:30,067 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=64ddf686fed30cdb23b2e874efec598d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:30,067 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=64ddf686fed30cdb23b2e874efec598d, ASSIGN 2024-11-08T05:13:30,067 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 868527ec322eba1d7cbd6371701f7ad2, server=d1ba675678c9,41485,1731042802970 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,067 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9b3274d80dcb7deed617c19ac1de4a68, server=d1ba675678c9,41485,1731042802970 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,067 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 81806cb184c70c9dc0df408694092d85, server=d1ba675678c9,39291,1731042803020 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,067 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 64ddf686fed30cdb23b2e874efec598d, server=d1ba675678c9,39291,1731042803020 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,067 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 445b14856951586918c91d4ba342f1ff, server=d1ba675678c9,41485,1731042802970 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,067 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=64, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure e61dc9e059921fddc5ad7f6f7971acc6, server=d1ba675678c9,41485,1731042802970:java.lang.RuntimeException: inject code bug; OpenRegionProcedure e61dc9e059921fddc5ad7f6f7971acc6, server=d1ba675678c9,41485,1731042802970 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,067 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 10 msec 2024-11-08T05:13:30,068 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37587 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T05:13:30,068 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=61 2024-11-08T05:13:30,068 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=d1ba675678c9,41485,1731042802970, table=hbase:meta, region=1588230740 2024-11-08T05:13:30,068 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37587 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T05:13:30,069 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d1ba675678c9,41485,1731042802970, state=OPEN 2024-11-08T05:13:30,081 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39291-0x10118dcefab0003, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T05:13:30,081 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41485-0x10118dcefab0002, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T05:13:30,081 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37587-0x10118dcefab0000, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T05:13:30,082 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T05:13:30,082 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T05:13:30,082 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T05:13:30,082 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e61dc9e059921fddc5ad7f6f7971acc6, ASSIGN to state=OFFLINE, location=null, table=test, region=e61dc9e059921fddc5ad7f6f7971acc6 to restore RIT 2024-11-08T05:13:30,083 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=81806cb184c70c9dc0df408694092d85, ASSIGN to state=OFFLINE, location=null, table=test, region=81806cb184c70c9dc0df408694092d85 to restore RIT 2024-11-08T05:13:30,083 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=868527ec322eba1d7cbd6371701f7ad2, ASSIGN to state=OFFLINE, location=null, table=test, region=868527ec322eba1d7cbd6371701f7ad2 to restore RIT 2024-11-08T05:13:30,083 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a12ea80f78b339cfbfc0e18b5338df0c, ASSIGN to state=OFFLINE, location=null, table=test, region=a12ea80f78b339cfbfc0e18b5338df0c to restore RIT 2024-11-08T05:13:30,083 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=445b14856951586918c91d4ba342f1ff, ASSIGN to state=OFFLINE, location=null, table=test, region=445b14856951586918c91d4ba342f1ff to restore RIT 2024-11-08T05:13:30,083 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9b3274d80dcb7deed617c19ac1de4a68, ASSIGN to state=OFFLINE, location=null, table=test, region=9b3274d80dcb7deed617c19ac1de4a68 to restore RIT 2024-11-08T05:13:30,083 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7830dc6b1acea6e1d468f8008d8a82b9, ASSIGN to state=OFFLINE, location=null, table=test, region=7830dc6b1acea6e1d468f8008d8a82b9 to restore RIT 2024-11-08T05:13:30,083 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=64ddf686fed30cdb23b2e874efec598d, ASSIGN to state=OFFLINE, location=null, table=test, region=64ddf686fed30cdb23b2e874efec598d to restore RIT 2024-11-08T05:13:30,083 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731042840083 2024-11-08T05:13:30,083 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-08T05:13:30,083 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure e61dc9e059921fddc5ad7f6f7971acc6, server=d1ba675678c9,41485,1731042802970:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d1ba675678c9,46753,1731042802851, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T05:13:30,085 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-08T05:13:30,086 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-08T05:13:30,086 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 445b14856951586918c91d4ba342f1ff, server=d1ba675678c9,41485,1731042802970 2024-11-08T05:13:30,086 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T05:13:30,086 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 445b14856951586918c91d4ba342f1ff, server=d1ba675678c9,41485,1731042802970 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T05:13:30,092 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6240e38715889f3bb77bb89478c81a6c, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,092 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eeaad0e7e7aa66f0b51c273c0aaf953c, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,093 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=68bbd0353f97ee12228f7798b0b987ea, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,093 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=64ddf686fed30cdb23b2e874efec598d, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,093 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d7680a9929f15640ca61551f186f33c1, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,093 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=65d5312dbd2f96bf3b16bdfd7504dff6, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,094 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9b3274d80dcb7deed617c19ac1de4a68, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,094 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2d23e3e11ecc480e314e989d1bb8d0c3, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,094 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fe1545b16a81758dcb9430dbf98c4ab9, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,094 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a12ea80f78b339cfbfc0e18b5338df0c, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,094 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e1c63297d3a8aea2fdbd1842301c811, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,095 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=40440b5f39bb5421276aa0b156481a9e, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,095 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=27c9aaf34aaf98cd5bc8030630ae7fe3, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,095 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=868527ec322eba1d7cbd6371701f7ad2, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,095 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d951febea78018bb9f10f07d8788575d, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,096 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b0ebd9a9794acc56e430b36d1f17621c, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,096 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7830dc6b1acea6e1d468f8008d8a82b9, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,096 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12f1148a7b8db3e4f515337b8b7746fa, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,096 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e26dcb2339b364f3699d110c15cd4bb3, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,096 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=445b14856951586918c91d4ba342f1ff, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,097 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=14efcc1b9822cc74f4d4a96648555167, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,097 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7c8f72849b2e1719ed903afc73e0571a, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,097 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=81806cb184c70c9dc0df408694092d85, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,097 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=87b3455cffd1f4e06e5901b3bcbed9dc, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,098 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=de358f808110a48fa32e0533fefdc95c, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,098 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e61dc9e059921fddc5ad7f6f7971acc6, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,098 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-08T05:13:30,098 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731042870098 2024-11-08T05:13:30,098 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731042930098 2024-11-08T05:13:30,098 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 14 msec 2024-11-08T05:13:30,098 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-08T05:13:30,098 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T05:13:30,098 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-08T05:13:30,098 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-08T05:13:30,099 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-08T05:13:30,099 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-08T05:13:30,099 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-08T05:13:30,099 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-08T05:13:30,100 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-08T05:13:30,105 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-08T05:13:30,105 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T05:13:30,105 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T05:13:30,105 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T05:13:30,105 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T05:13:30,105 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T05:13:30,105 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T05:13:30,105 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T05:13:30,105 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T05:13:30,105 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T05:13:30,105 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T05:13:30,105 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T05:13:30,105 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T05:13:30,105 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T05:13:30,105 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T05:13:30,105 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T05:13:30,106 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T05:13:30,106 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T05:13:30,106 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T05:13:30,106 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T05:13:30,106 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-08T05:13:30,107 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-08T05:13:30,111 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure e61dc9e059921fddc5ad7f6f7971acc6, server=d1ba675678c9,41485,1731042802970:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d1ba675678c9,46753,1731042802851, splitWal=true, meta=true 2024-11-08T05:13:30,111 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure d1ba675678c9%2C46753%2C1731042802851.meta.1731042805109.meta, worker=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:30,111 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d1ba675678c9%2C46753%2C1731042802851.meta.1731042805109.meta, worker=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:30,111 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T05:13:30,111 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d1ba675678c9,41485,1731042802970 2024-11-08T05:13:30,111 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure d1ba675678c9%2C46753%2C1731042802851.1731042804603, worker=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:30,111 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d1ba675678c9%2C46753%2C1731042802851.1731042804603, worker=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:30,111 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e61dc9e059921fddc5ad7f6f7971acc6, ASSIGN 2024-11-08T05:13:30,111 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=81806cb184c70c9dc0df408694092d85, ASSIGN 2024-11-08T05:13:30,111 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=868527ec322eba1d7cbd6371701f7ad2, ASSIGN 2024-11-08T05:13:30,112 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a12ea80f78b339cfbfc0e18b5338df0c, ASSIGN 2024-11-08T05:13:30,112 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=445b14856951586918c91d4ba342f1ff, ASSIGN 2024-11-08T05:13:30,112 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9b3274d80dcb7deed617c19ac1de4a68, ASSIGN 2024-11-08T05:13:30,112 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7830dc6b1acea6e1d468f8008d8a82b9, ASSIGN 2024-11-08T05:13:30,112 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=64ddf686fed30cdb23b2e874efec598d, ASSIGN 2024-11-08T05:13:30,112 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 868527ec322eba1d7cbd6371701f7ad2, server=d1ba675678c9,41485,1731042802970 2024-11-08T05:13:30,112 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9b3274d80dcb7deed617c19ac1de4a68, server=d1ba675678c9,41485,1731042802970 2024-11-08T05:13:30,112 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 81806cb184c70c9dc0df408694092d85, server=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:30,112 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 64ddf686fed30cdb23b2e874efec598d, server=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:30,112 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 445b14856951586918c91d4ba342f1ff, server=d1ba675678c9,41485,1731042802970 2024-11-08T05:13:30,112 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure e61dc9e059921fddc5ad7f6f7971acc6, server=d1ba675678c9,41485,1731042802970:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d1ba675678c9,46753,1731042802851, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:30,112 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure d1ba675678c9%2C46753%2C1731042802851.meta.1731042805109.meta, worker=d1ba675678c9,39291,1731042803020 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,112 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d1ba675678c9%2C46753%2C1731042802851.meta.1731042805109.meta, worker=d1ba675678c9,39291,1731042803020 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,112 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,112 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d1ba675678c9,41485,1731042802970 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,113 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure d1ba675678c9%2C46753%2C1731042802851.1731042804603, worker=d1ba675678c9,39291,1731042803020 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,113 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d1ba675678c9%2C46753%2C1731042802851.1731042804603, worker=d1ba675678c9,39291,1731042803020 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,113 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e61dc9e059921fddc5ad7f6f7971acc6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:30,113 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e61dc9e059921fddc5ad7f6f7971acc6, ASSIGN 2024-11-08T05:13:30,113 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=81806cb184c70c9dc0df408694092d85, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:30,113 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=81806cb184c70c9dc0df408694092d85, ASSIGN 2024-11-08T05:13:30,113 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=868527ec322eba1d7cbd6371701f7ad2, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:30,113 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=868527ec322eba1d7cbd6371701f7ad2, ASSIGN 2024-11-08T05:13:30,113 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a12ea80f78b339cfbfc0e18b5338df0c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:30,113 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a12ea80f78b339cfbfc0e18b5338df0c, ASSIGN 2024-11-08T05:13:30,113 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=445b14856951586918c91d4ba342f1ff, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:30,113 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=445b14856951586918c91d4ba342f1ff, ASSIGN 2024-11-08T05:13:30,113 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9b3274d80dcb7deed617c19ac1de4a68, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:30,113 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9b3274d80dcb7deed617c19ac1de4a68, ASSIGN 2024-11-08T05:13:30,113 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7830dc6b1acea6e1d468f8008d8a82b9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:30,113 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7830dc6b1acea6e1d468f8008d8a82b9, ASSIGN 2024-11-08T05:13:30,113 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=64ddf686fed30cdb23b2e874efec598d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:30,113 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=64ddf686fed30cdb23b2e874efec598d, ASSIGN 2024-11-08T05:13:30,113 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 868527ec322eba1d7cbd6371701f7ad2, server=d1ba675678c9,41485,1731042802970 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,113 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9b3274d80dcb7deed617c19ac1de4a68, server=d1ba675678c9,41485,1731042802970 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,114 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 81806cb184c70c9dc0df408694092d85, server=d1ba675678c9,39291,1731042803020 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,114 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 64ddf686fed30cdb23b2e874efec598d, server=d1ba675678c9,39291,1731042803020 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,114 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 445b14856951586918c91d4ba342f1ff, server=d1ba675678c9,41485,1731042802970 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,114 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 13 msec 2024-11-08T05:13:30,115 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=61 2024-11-08T05:13:30,115 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=d1ba675678c9,41485,1731042802970, table=hbase:meta, region=1588230740 2024-11-08T05:13:30,115 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d1ba675678c9,41485,1731042802970, state=OPEN 2024-11-08T05:13:30,123 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39291-0x10118dcefab0003, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T05:13:30,124 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37587-0x10118dcefab0000, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T05:13:30,124 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41485-0x10118dcefab0002, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T05:13:30,124 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T05:13:30,124 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T05:13:30,124 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T05:13:30,124 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e61dc9e059921fddc5ad7f6f7971acc6, ASSIGN to state=OFFLINE, location=null, table=test, region=e61dc9e059921fddc5ad7f6f7971acc6 to restore RIT 2024-11-08T05:13:30,124 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=81806cb184c70c9dc0df408694092d85, ASSIGN to state=OFFLINE, location=null, table=test, region=81806cb184c70c9dc0df408694092d85 to restore RIT 2024-11-08T05:13:30,124 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=868527ec322eba1d7cbd6371701f7ad2, ASSIGN to state=OFFLINE, location=null, table=test, region=868527ec322eba1d7cbd6371701f7ad2 to restore RIT 2024-11-08T05:13:30,125 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a12ea80f78b339cfbfc0e18b5338df0c, ASSIGN to state=OFFLINE, location=null, table=test, region=a12ea80f78b339cfbfc0e18b5338df0c to restore RIT 2024-11-08T05:13:30,125 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=445b14856951586918c91d4ba342f1ff, ASSIGN to state=OFFLINE, location=null, table=test, region=445b14856951586918c91d4ba342f1ff to restore RIT 2024-11-08T05:13:30,125 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9b3274d80dcb7deed617c19ac1de4a68, ASSIGN to state=OFFLINE, location=null, table=test, region=9b3274d80dcb7deed617c19ac1de4a68 to restore RIT 2024-11-08T05:13:30,125 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7830dc6b1acea6e1d468f8008d8a82b9, ASSIGN to state=OFFLINE, location=null, table=test, region=7830dc6b1acea6e1d468f8008d8a82b9 to restore RIT 2024-11-08T05:13:30,125 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=64ddf686fed30cdb23b2e874efec598d, ASSIGN to state=OFFLINE, location=null, table=test, region=64ddf686fed30cdb23b2e874efec598d to restore RIT 2024-11-08T05:13:30,125 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731042840125 2024-11-08T05:13:30,125 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-08T05:13:30,125 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure e61dc9e059921fddc5ad7f6f7971acc6, server=d1ba675678c9,41485,1731042802970:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d1ba675678c9,46753,1731042802851, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T05:13:30,127 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-08T05:13:30,128 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-08T05:13:30,128 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=75, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 64ddf686fed30cdb23b2e874efec598d, server=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:30,128 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T05:13:30,128 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=75, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 64ddf686fed30cdb23b2e874efec598d, server=d1ba675678c9,39291,1731042803020 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T05:13:30,132 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6240e38715889f3bb77bb89478c81a6c, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,132 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eeaad0e7e7aa66f0b51c273c0aaf953c, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,132 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=68bbd0353f97ee12228f7798b0b987ea, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,133 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=64ddf686fed30cdb23b2e874efec598d, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,133 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d7680a9929f15640ca61551f186f33c1, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,133 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=65d5312dbd2f96bf3b16bdfd7504dff6, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,134 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9b3274d80dcb7deed617c19ac1de4a68, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,134 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2d23e3e11ecc480e314e989d1bb8d0c3, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,134 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fe1545b16a81758dcb9430dbf98c4ab9, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,134 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a12ea80f78b339cfbfc0e18b5338df0c, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,134 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e1c63297d3a8aea2fdbd1842301c811, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,135 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=40440b5f39bb5421276aa0b156481a9e, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,135 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=27c9aaf34aaf98cd5bc8030630ae7fe3, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,135 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=868527ec322eba1d7cbd6371701f7ad2, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,135 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d951febea78018bb9f10f07d8788575d, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,136 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b0ebd9a9794acc56e430b36d1f17621c, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,136 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7830dc6b1acea6e1d468f8008d8a82b9, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,136 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12f1148a7b8db3e4f515337b8b7746fa, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,136 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e26dcb2339b364f3699d110c15cd4bb3, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,136 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=445b14856951586918c91d4ba342f1ff, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,137 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=14efcc1b9822cc74f4d4a96648555167, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,137 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7c8f72849b2e1719ed903afc73e0571a, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,137 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=81806cb184c70c9dc0df408694092d85, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,137 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=87b3455cffd1f4e06e5901b3bcbed9dc, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,137 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=de358f808110a48fa32e0533fefdc95c, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,138 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e61dc9e059921fddc5ad7f6f7971acc6, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,138 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-08T05:13:30,138 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731042870138 2024-11-08T05:13:30,138 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731042930138 2024-11-08T05:13:30,138 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 12 msec 2024-11-08T05:13:30,138 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-08T05:13:30,138 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T05:13:30,138 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-08T05:13:30,138 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-08T05:13:30,139 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-08T05:13:30,139 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-08T05:13:30,139 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-08T05:13:30,139 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-08T05:13:30,140 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-08T05:13:30,143 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-08T05:13:30,143 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T05:13:30,143 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T05:13:30,143 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T05:13:30,143 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T05:13:30,144 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T05:13:30,144 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T05:13:30,144 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T05:13:30,144 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T05:13:30,144 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T05:13:30,144 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T05:13:30,144 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T05:13:30,144 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T05:13:30,144 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T05:13:30,144 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T05:13:30,144 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T05:13:30,144 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T05:13:30,144 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T05:13:30,144 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T05:13:30,144 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-08T05:13:30,145 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-08T05:13:30,147 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure e61dc9e059921fddc5ad7f6f7971acc6, server=d1ba675678c9,41485,1731042802970:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d1ba675678c9,46753,1731042802851, splitWal=true, meta=true 2024-11-08T05:13:30,147 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure d1ba675678c9%2C46753%2C1731042802851.meta.1731042805109.meta, worker=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:30,147 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d1ba675678c9%2C46753%2C1731042802851.meta.1731042805109.meta, worker=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:30,148 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T05:13:30,148 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d1ba675678c9,41485,1731042802970 2024-11-08T05:13:30,148 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure d1ba675678c9%2C46753%2C1731042802851.1731042804603, worker=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:30,148 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d1ba675678c9%2C46753%2C1731042802851.1731042804603, worker=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:30,148 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e61dc9e059921fddc5ad7f6f7971acc6, ASSIGN 2024-11-08T05:13:30,148 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=81806cb184c70c9dc0df408694092d85, ASSIGN 2024-11-08T05:13:30,148 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=868527ec322eba1d7cbd6371701f7ad2, ASSIGN 2024-11-08T05:13:30,148 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a12ea80f78b339cfbfc0e18b5338df0c, ASSIGN 2024-11-08T05:13:30,148 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=445b14856951586918c91d4ba342f1ff, ASSIGN 2024-11-08T05:13:30,148 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9b3274d80dcb7deed617c19ac1de4a68, ASSIGN 2024-11-08T05:13:30,148 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7830dc6b1acea6e1d468f8008d8a82b9, ASSIGN 2024-11-08T05:13:30,148 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=64ddf686fed30cdb23b2e874efec598d, ASSIGN 2024-11-08T05:13:30,148 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 868527ec322eba1d7cbd6371701f7ad2, server=d1ba675678c9,41485,1731042802970 2024-11-08T05:13:30,148 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9b3274d80dcb7deed617c19ac1de4a68, server=d1ba675678c9,41485,1731042802970 2024-11-08T05:13:30,148 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 81806cb184c70c9dc0df408694092d85, server=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:30,148 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 64ddf686fed30cdb23b2e874efec598d, server=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:30,148 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure e61dc9e059921fddc5ad7f6f7971acc6, server=d1ba675678c9,41485,1731042802970:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d1ba675678c9,46753,1731042802851, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:30,148 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure d1ba675678c9%2C46753%2C1731042802851.meta.1731042805109.meta, worker=d1ba675678c9,39291,1731042803020 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,148 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d1ba675678c9%2C46753%2C1731042802851.meta.1731042805109.meta, worker=d1ba675678c9,39291,1731042803020 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,148 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,149 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d1ba675678c9,41485,1731042802970 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,149 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure d1ba675678c9%2C46753%2C1731042802851.1731042804603, worker=d1ba675678c9,39291,1731042803020 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,149 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d1ba675678c9%2C46753%2C1731042802851.1731042804603, worker=d1ba675678c9,39291,1731042803020 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,149 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e61dc9e059921fddc5ad7f6f7971acc6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:30,149 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e61dc9e059921fddc5ad7f6f7971acc6, ASSIGN 2024-11-08T05:13:30,149 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=81806cb184c70c9dc0df408694092d85, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:30,149 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=81806cb184c70c9dc0df408694092d85, ASSIGN 2024-11-08T05:13:30,149 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=868527ec322eba1d7cbd6371701f7ad2, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:30,149 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=868527ec322eba1d7cbd6371701f7ad2, ASSIGN 2024-11-08T05:13:30,149 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a12ea80f78b339cfbfc0e18b5338df0c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:30,149 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a12ea80f78b339cfbfc0e18b5338df0c, ASSIGN 2024-11-08T05:13:30,149 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=445b14856951586918c91d4ba342f1ff, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:30,149 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=445b14856951586918c91d4ba342f1ff, ASSIGN 2024-11-08T05:13:30,149 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9b3274d80dcb7deed617c19ac1de4a68, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:30,150 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9b3274d80dcb7deed617c19ac1de4a68, ASSIGN 2024-11-08T05:13:30,150 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7830dc6b1acea6e1d468f8008d8a82b9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:30,150 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7830dc6b1acea6e1d468f8008d8a82b9, ASSIGN 2024-11-08T05:13:30,150 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=64ddf686fed30cdb23b2e874efec598d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:30,150 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=64ddf686fed30cdb23b2e874efec598d, ASSIGN 2024-11-08T05:13:30,150 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 868527ec322eba1d7cbd6371701f7ad2, server=d1ba675678c9,41485,1731042802970 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,150 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9b3274d80dcb7deed617c19ac1de4a68, server=d1ba675678c9,41485,1731042802970 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,150 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 81806cb184c70c9dc0df408694092d85, server=d1ba675678c9,39291,1731042803020 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,150 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 64ddf686fed30cdb23b2e874efec598d, server=d1ba675678c9,39291,1731042803020 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,150 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 10 msec 2024-11-08T05:13:30,151 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=61 2024-11-08T05:13:30,151 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=d1ba675678c9,41485,1731042802970, table=hbase:meta, region=1588230740 2024-11-08T05:13:30,152 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d1ba675678c9,41485,1731042802970, state=OPEN 2024-11-08T05:13:30,162 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41485-0x10118dcefab0002, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T05:13:30,162 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39291-0x10118dcefab0003, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T05:13:30,162 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37587-0x10118dcefab0000, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T05:13:30,162 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T05:13:30,162 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T05:13:30,162 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T05:13:30,163 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e61dc9e059921fddc5ad7f6f7971acc6, ASSIGN to state=OFFLINE, location=null, table=test, region=e61dc9e059921fddc5ad7f6f7971acc6 to restore RIT 2024-11-08T05:13:30,163 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=81806cb184c70c9dc0df408694092d85, ASSIGN to state=OFFLINE, location=null, table=test, region=81806cb184c70c9dc0df408694092d85 to restore RIT 2024-11-08T05:13:30,163 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=868527ec322eba1d7cbd6371701f7ad2, ASSIGN to state=OFFLINE, location=null, table=test, region=868527ec322eba1d7cbd6371701f7ad2 to restore RIT 2024-11-08T05:13:30,163 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a12ea80f78b339cfbfc0e18b5338df0c, ASSIGN to state=OFFLINE, location=null, table=test, region=a12ea80f78b339cfbfc0e18b5338df0c to restore RIT 2024-11-08T05:13:30,163 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=445b14856951586918c91d4ba342f1ff, ASSIGN to state=OFFLINE, location=null, table=test, region=445b14856951586918c91d4ba342f1ff to restore RIT 2024-11-08T05:13:30,163 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9b3274d80dcb7deed617c19ac1de4a68, ASSIGN to state=OFFLINE, location=null, table=test, region=9b3274d80dcb7deed617c19ac1de4a68 to restore RIT 2024-11-08T05:13:30,163 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7830dc6b1acea6e1d468f8008d8a82b9, ASSIGN to state=OFFLINE, location=null, table=test, region=7830dc6b1acea6e1d468f8008d8a82b9 to restore RIT 2024-11-08T05:13:30,163 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=64ddf686fed30cdb23b2e874efec598d, ASSIGN to state=OFFLINE, location=null, table=test, region=64ddf686fed30cdb23b2e874efec598d to restore RIT 2024-11-08T05:13:30,164 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731042840164 2024-11-08T05:13:30,164 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-08T05:13:30,164 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure e61dc9e059921fddc5ad7f6f7971acc6, server=d1ba675678c9,41485,1731042802970:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d1ba675678c9,46753,1731042802851, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T05:13:30,165 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-08T05:13:30,166 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-08T05:13:30,167 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=74, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 81806cb184c70c9dc0df408694092d85, server=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:30,167 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T05:13:30,167 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=74, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 81806cb184c70c9dc0df408694092d85, server=d1ba675678c9,39291,1731042803020 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T05:13:30,170 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6240e38715889f3bb77bb89478c81a6c, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,170 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eeaad0e7e7aa66f0b51c273c0aaf953c, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,170 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=68bbd0353f97ee12228f7798b0b987ea, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,171 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=64ddf686fed30cdb23b2e874efec598d, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,171 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d7680a9929f15640ca61551f186f33c1, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,171 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=65d5312dbd2f96bf3b16bdfd7504dff6, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,171 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9b3274d80dcb7deed617c19ac1de4a68, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,171 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2d23e3e11ecc480e314e989d1bb8d0c3, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,172 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fe1545b16a81758dcb9430dbf98c4ab9, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,172 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a12ea80f78b339cfbfc0e18b5338df0c, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,172 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e1c63297d3a8aea2fdbd1842301c811, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,172 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=40440b5f39bb5421276aa0b156481a9e, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,172 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=27c9aaf34aaf98cd5bc8030630ae7fe3, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,172 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=868527ec322eba1d7cbd6371701f7ad2, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,173 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d951febea78018bb9f10f07d8788575d, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,173 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b0ebd9a9794acc56e430b36d1f17621c, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,173 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7830dc6b1acea6e1d468f8008d8a82b9, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,173 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12f1148a7b8db3e4f515337b8b7746fa, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,173 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e26dcb2339b364f3699d110c15cd4bb3, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,174 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=445b14856951586918c91d4ba342f1ff, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,174 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=14efcc1b9822cc74f4d4a96648555167, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,174 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7c8f72849b2e1719ed903afc73e0571a, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,174 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=81806cb184c70c9dc0df408694092d85, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,174 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=87b3455cffd1f4e06e5901b3bcbed9dc, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,174 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=de358f808110a48fa32e0533fefdc95c, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,175 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e61dc9e059921fddc5ad7f6f7971acc6, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,175 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-08T05:13:30,175 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731042870175 2024-11-08T05:13:30,175 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731042930175 2024-11-08T05:13:30,175 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 11 msec 2024-11-08T05:13:30,175 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-08T05:13:30,175 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T05:13:30,175 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-08T05:13:30,175 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-08T05:13:30,176 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-08T05:13:30,176 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-08T05:13:30,176 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-08T05:13:30,176 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-08T05:13:30,177 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-08T05:13:30,180 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-08T05:13:30,180 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T05:13:30,180 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T05:13:30,180 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T05:13:30,180 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T05:13:30,180 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T05:13:30,180 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T05:13:30,181 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T05:13:30,181 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T05:13:30,181 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T05:13:30,181 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T05:13:30,181 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T05:13:30,181 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T05:13:30,181 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T05:13:30,181 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T05:13:30,181 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T05:13:30,181 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T05:13:30,181 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T05:13:30,181 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-08T05:13:30,182 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-08T05:13:30,184 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37587 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T05:13:30,184 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37587 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T05:13:30,185 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure e61dc9e059921fddc5ad7f6f7971acc6, server=d1ba675678c9,41485,1731042802970:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d1ba675678c9,46753,1731042802851, splitWal=true, meta=true 2024-11-08T05:13:30,185 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure d1ba675678c9%2C46753%2C1731042802851.meta.1731042805109.meta, worker=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:30,185 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d1ba675678c9%2C46753%2C1731042802851.meta.1731042805109.meta, worker=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:30,185 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T05:13:30,185 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d1ba675678c9,41485,1731042802970 2024-11-08T05:13:30,185 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure d1ba675678c9%2C46753%2C1731042802851.1731042804603, worker=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:30,185 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d1ba675678c9%2C46753%2C1731042802851.1731042804603, worker=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:30,185 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e61dc9e059921fddc5ad7f6f7971acc6, ASSIGN 2024-11-08T05:13:30,185 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=81806cb184c70c9dc0df408694092d85, ASSIGN 2024-11-08T05:13:30,186 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=868527ec322eba1d7cbd6371701f7ad2, ASSIGN 2024-11-08T05:13:30,186 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a12ea80f78b339cfbfc0e18b5338df0c, ASSIGN 2024-11-08T05:13:30,186 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=445b14856951586918c91d4ba342f1ff, ASSIGN 2024-11-08T05:13:30,186 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9b3274d80dcb7deed617c19ac1de4a68, ASSIGN 2024-11-08T05:13:30,186 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7830dc6b1acea6e1d468f8008d8a82b9, ASSIGN 2024-11-08T05:13:30,186 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=64ddf686fed30cdb23b2e874efec598d, ASSIGN 2024-11-08T05:13:30,186 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 868527ec322eba1d7cbd6371701f7ad2, server=d1ba675678c9,41485,1731042802970 2024-11-08T05:13:30,186 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9b3274d80dcb7deed617c19ac1de4a68, server=d1ba675678c9,41485,1731042802970 2024-11-08T05:13:30,186 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 81806cb184c70c9dc0df408694092d85, server=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:30,186 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure e61dc9e059921fddc5ad7f6f7971acc6, server=d1ba675678c9,41485,1731042802970:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d1ba675678c9,46753,1731042802851, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:30,186 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure d1ba675678c9%2C46753%2C1731042802851.meta.1731042805109.meta, worker=d1ba675678c9,39291,1731042803020 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,186 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d1ba675678c9%2C46753%2C1731042802851.meta.1731042805109.meta, worker=d1ba675678c9,39291,1731042803020 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,186 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,186 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d1ba675678c9,41485,1731042802970 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,186 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure d1ba675678c9%2C46753%2C1731042802851.1731042804603, worker=d1ba675678c9,39291,1731042803020 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,187 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d1ba675678c9%2C46753%2C1731042802851.1731042804603, worker=d1ba675678c9,39291,1731042803020 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,187 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e61dc9e059921fddc5ad7f6f7971acc6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:30,187 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e61dc9e059921fddc5ad7f6f7971acc6, ASSIGN 2024-11-08T05:13:30,187 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=81806cb184c70c9dc0df408694092d85, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:30,187 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=81806cb184c70c9dc0df408694092d85, ASSIGN 2024-11-08T05:13:30,187 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=868527ec322eba1d7cbd6371701f7ad2, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:30,187 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=868527ec322eba1d7cbd6371701f7ad2, ASSIGN 2024-11-08T05:13:30,187 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a12ea80f78b339cfbfc0e18b5338df0c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:30,187 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a12ea80f78b339cfbfc0e18b5338df0c, ASSIGN 2024-11-08T05:13:30,187 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=445b14856951586918c91d4ba342f1ff, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:30,187 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=445b14856951586918c91d4ba342f1ff, ASSIGN 2024-11-08T05:13:30,187 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9b3274d80dcb7deed617c19ac1de4a68, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:30,187 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9b3274d80dcb7deed617c19ac1de4a68, ASSIGN 2024-11-08T05:13:30,187 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7830dc6b1acea6e1d468f8008d8a82b9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:30,187 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7830dc6b1acea6e1d468f8008d8a82b9, ASSIGN 2024-11-08T05:13:30,187 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=64ddf686fed30cdb23b2e874efec598d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:30,188 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=64ddf686fed30cdb23b2e874efec598d, ASSIGN 2024-11-08T05:13:30,188 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 868527ec322eba1d7cbd6371701f7ad2, server=d1ba675678c9,41485,1731042802970 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,188 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9b3274d80dcb7deed617c19ac1de4a68, server=d1ba675678c9,41485,1731042802970 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,188 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 81806cb184c70c9dc0df408694092d85, server=d1ba675678c9,39291,1731042803020 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,188 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 10 msec 2024-11-08T05:13:30,189 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=61 2024-11-08T05:13:30,189 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=d1ba675678c9,41485,1731042802970, table=hbase:meta, region=1588230740 2024-11-08T05:13:30,189 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d1ba675678c9,41485,1731042802970, state=OPEN 2024-11-08T05:13:30,197 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41485-0x10118dcefab0002, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T05:13:30,197 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37587-0x10118dcefab0000, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T05:13:30,197 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39291-0x10118dcefab0003, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T05:13:30,197 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T05:13:30,197 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T05:13:30,197 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T05:13:30,198 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e61dc9e059921fddc5ad7f6f7971acc6, ASSIGN to state=OFFLINE, location=null, table=test, region=e61dc9e059921fddc5ad7f6f7971acc6 to restore RIT 2024-11-08T05:13:30,198 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=81806cb184c70c9dc0df408694092d85, ASSIGN to state=OFFLINE, location=null, table=test, region=81806cb184c70c9dc0df408694092d85 to restore RIT 2024-11-08T05:13:30,198 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=868527ec322eba1d7cbd6371701f7ad2, ASSIGN to state=OFFLINE, location=null, table=test, region=868527ec322eba1d7cbd6371701f7ad2 to restore RIT 2024-11-08T05:13:30,198 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a12ea80f78b339cfbfc0e18b5338df0c, ASSIGN to state=OFFLINE, location=null, table=test, region=a12ea80f78b339cfbfc0e18b5338df0c to restore RIT 2024-11-08T05:13:30,198 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=445b14856951586918c91d4ba342f1ff, ASSIGN to state=OFFLINE, location=null, table=test, region=445b14856951586918c91d4ba342f1ff to restore RIT 2024-11-08T05:13:30,198 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9b3274d80dcb7deed617c19ac1de4a68, ASSIGN to state=OFFLINE, location=null, table=test, region=9b3274d80dcb7deed617c19ac1de4a68 to restore RIT 2024-11-08T05:13:30,198 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7830dc6b1acea6e1d468f8008d8a82b9, ASSIGN to state=OFFLINE, location=null, table=test, region=7830dc6b1acea6e1d468f8008d8a82b9 to restore RIT 2024-11-08T05:13:30,198 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=64ddf686fed30cdb23b2e874efec598d, ASSIGN to state=OFFLINE, location=null, table=test, region=64ddf686fed30cdb23b2e874efec598d to restore RIT 2024-11-08T05:13:30,199 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731042840199 2024-11-08T05:13:30,199 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-08T05:13:30,199 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure e61dc9e059921fddc5ad7f6f7971acc6, server=d1ba675678c9,41485,1731042802970:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d1ba675678c9,46753,1731042802851, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T05:13:30,201 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-08T05:13:30,202 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-08T05:13:30,202 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=73, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9b3274d80dcb7deed617c19ac1de4a68, server=d1ba675678c9,41485,1731042802970 2024-11-08T05:13:30,202 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T05:13:30,202 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=73, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9b3274d80dcb7deed617c19ac1de4a68, server=d1ba675678c9,41485,1731042802970 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T05:13:30,205 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6240e38715889f3bb77bb89478c81a6c, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,205 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eeaad0e7e7aa66f0b51c273c0aaf953c, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,205 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=68bbd0353f97ee12228f7798b0b987ea, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,206 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=64ddf686fed30cdb23b2e874efec598d, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,206 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d7680a9929f15640ca61551f186f33c1, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,206 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=65d5312dbd2f96bf3b16bdfd7504dff6, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,206 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9b3274d80dcb7deed617c19ac1de4a68, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,206 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2d23e3e11ecc480e314e989d1bb8d0c3, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,206 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fe1545b16a81758dcb9430dbf98c4ab9, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,207 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a12ea80f78b339cfbfc0e18b5338df0c, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,207 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e1c63297d3a8aea2fdbd1842301c811, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,207 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=40440b5f39bb5421276aa0b156481a9e, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,207 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=27c9aaf34aaf98cd5bc8030630ae7fe3, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,207 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=868527ec322eba1d7cbd6371701f7ad2, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,207 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d951febea78018bb9f10f07d8788575d, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,207 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b0ebd9a9794acc56e430b36d1f17621c, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,207 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7830dc6b1acea6e1d468f8008d8a82b9, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,208 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12f1148a7b8db3e4f515337b8b7746fa, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,208 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e26dcb2339b364f3699d110c15cd4bb3, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,208 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=445b14856951586918c91d4ba342f1ff, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,208 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=14efcc1b9822cc74f4d4a96648555167, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,208 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7c8f72849b2e1719ed903afc73e0571a, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,208 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=81806cb184c70c9dc0df408694092d85, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,208 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=87b3455cffd1f4e06e5901b3bcbed9dc, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,209 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=de358f808110a48fa32e0533fefdc95c, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,209 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e61dc9e059921fddc5ad7f6f7971acc6, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,209 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-08T05:13:30,209 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731042870209 2024-11-08T05:13:30,209 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731042930209 2024-11-08T05:13:30,209 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 10 msec 2024-11-08T05:13:30,209 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-08T05:13:30,209 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T05:13:30,209 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-08T05:13:30,209 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-08T05:13:30,209 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-08T05:13:30,209 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-08T05:13:30,210 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-08T05:13:30,210 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-08T05:13:30,211 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-08T05:13:30,214 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-08T05:13:30,214 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T05:13:30,214 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T05:13:30,214 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T05:13:30,214 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T05:13:30,214 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T05:13:30,214 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T05:13:30,214 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T05:13:30,214 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T05:13:30,214 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T05:13:30,214 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T05:13:30,214 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T05:13:30,214 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T05:13:30,214 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T05:13:30,215 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T05:13:30,215 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T05:13:30,215 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T05:13:30,215 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-08T05:13:30,216 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-08T05:13:30,218 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure e61dc9e059921fddc5ad7f6f7971acc6, server=d1ba675678c9,41485,1731042802970:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d1ba675678c9,46753,1731042802851, splitWal=true, meta=true 2024-11-08T05:13:30,218 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure d1ba675678c9%2C46753%2C1731042802851.meta.1731042805109.meta, worker=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:30,218 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d1ba675678c9%2C46753%2C1731042802851.meta.1731042805109.meta, worker=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:30,218 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T05:13:30,219 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d1ba675678c9,41485,1731042802970 2024-11-08T05:13:30,219 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure d1ba675678c9%2C46753%2C1731042802851.1731042804603, worker=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:30,219 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d1ba675678c9%2C46753%2C1731042802851.1731042804603, worker=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:30,219 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e61dc9e059921fddc5ad7f6f7971acc6, ASSIGN 2024-11-08T05:13:30,219 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=81806cb184c70c9dc0df408694092d85, ASSIGN 2024-11-08T05:13:30,219 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=868527ec322eba1d7cbd6371701f7ad2, ASSIGN 2024-11-08T05:13:30,219 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a12ea80f78b339cfbfc0e18b5338df0c, ASSIGN 2024-11-08T05:13:30,219 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=445b14856951586918c91d4ba342f1ff, ASSIGN 2024-11-08T05:13:30,219 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9b3274d80dcb7deed617c19ac1de4a68, ASSIGN 2024-11-08T05:13:30,219 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7830dc6b1acea6e1d468f8008d8a82b9, ASSIGN 2024-11-08T05:13:30,219 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=64ddf686fed30cdb23b2e874efec598d, ASSIGN 2024-11-08T05:13:30,219 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 868527ec322eba1d7cbd6371701f7ad2, server=d1ba675678c9,41485,1731042802970 2024-11-08T05:13:30,219 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9b3274d80dcb7deed617c19ac1de4a68, server=d1ba675678c9,41485,1731042802970 2024-11-08T05:13:30,219 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure e61dc9e059921fddc5ad7f6f7971acc6, server=d1ba675678c9,41485,1731042802970:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d1ba675678c9,46753,1731042802851, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:30,219 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure d1ba675678c9%2C46753%2C1731042802851.meta.1731042805109.meta, worker=d1ba675678c9,39291,1731042803020 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,219 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d1ba675678c9%2C46753%2C1731042802851.meta.1731042805109.meta, worker=d1ba675678c9,39291,1731042803020 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,219 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,220 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d1ba675678c9,41485,1731042802970 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,220 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure d1ba675678c9%2C46753%2C1731042802851.1731042804603, worker=d1ba675678c9,39291,1731042803020 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,220 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d1ba675678c9%2C46753%2C1731042802851.1731042804603, worker=d1ba675678c9,39291,1731042803020 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,220 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e61dc9e059921fddc5ad7f6f7971acc6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:30,220 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e61dc9e059921fddc5ad7f6f7971acc6, ASSIGN 2024-11-08T05:13:30,220 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=81806cb184c70c9dc0df408694092d85, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:30,220 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=81806cb184c70c9dc0df408694092d85, ASSIGN 2024-11-08T05:13:30,220 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=868527ec322eba1d7cbd6371701f7ad2, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:30,220 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=868527ec322eba1d7cbd6371701f7ad2, ASSIGN 2024-11-08T05:13:30,220 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a12ea80f78b339cfbfc0e18b5338df0c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:30,220 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a12ea80f78b339cfbfc0e18b5338df0c, ASSIGN 2024-11-08T05:13:30,220 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=445b14856951586918c91d4ba342f1ff, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:30,220 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=445b14856951586918c91d4ba342f1ff, ASSIGN 2024-11-08T05:13:30,220 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9b3274d80dcb7deed617c19ac1de4a68, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:30,220 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9b3274d80dcb7deed617c19ac1de4a68, ASSIGN 2024-11-08T05:13:30,220 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7830dc6b1acea6e1d468f8008d8a82b9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:30,220 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7830dc6b1acea6e1d468f8008d8a82b9, ASSIGN 2024-11-08T05:13:30,221 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=64ddf686fed30cdb23b2e874efec598d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:30,221 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=64ddf686fed30cdb23b2e874efec598d, ASSIGN 2024-11-08T05:13:30,221 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 868527ec322eba1d7cbd6371701f7ad2, server=d1ba675678c9,41485,1731042802970 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,221 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9b3274d80dcb7deed617c19ac1de4a68, server=d1ba675678c9,41485,1731042802970 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,221 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 10 msec 2024-11-08T05:13:30,222 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=61 2024-11-08T05:13:30,222 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=d1ba675678c9,41485,1731042802970, table=hbase:meta, region=1588230740 2024-11-08T05:13:30,222 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d1ba675678c9,41485,1731042802970, state=OPEN 2024-11-08T05:13:30,229 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37587-0x10118dcefab0000, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T05:13:30,229 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41485-0x10118dcefab0002, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T05:13:30,229 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39291-0x10118dcefab0003, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T05:13:30,229 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T05:13:30,229 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T05:13:30,229 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T05:13:30,230 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e61dc9e059921fddc5ad7f6f7971acc6, ASSIGN to state=OFFLINE, location=null, table=test, region=e61dc9e059921fddc5ad7f6f7971acc6 to restore RIT 2024-11-08T05:13:30,230 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=81806cb184c70c9dc0df408694092d85, ASSIGN to state=OFFLINE, location=null, table=test, region=81806cb184c70c9dc0df408694092d85 to restore RIT 2024-11-08T05:13:30,230 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=868527ec322eba1d7cbd6371701f7ad2, ASSIGN to state=OFFLINE, location=null, table=test, region=868527ec322eba1d7cbd6371701f7ad2 to restore RIT 2024-11-08T05:13:30,230 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a12ea80f78b339cfbfc0e18b5338df0c, ASSIGN to state=OFFLINE, location=null, table=test, region=a12ea80f78b339cfbfc0e18b5338df0c to restore RIT 2024-11-08T05:13:30,230 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=445b14856951586918c91d4ba342f1ff, ASSIGN to state=OFFLINE, location=null, table=test, region=445b14856951586918c91d4ba342f1ff to restore RIT 2024-11-08T05:13:30,230 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9b3274d80dcb7deed617c19ac1de4a68, ASSIGN to state=OFFLINE, location=null, table=test, region=9b3274d80dcb7deed617c19ac1de4a68 to restore RIT 2024-11-08T05:13:30,230 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7830dc6b1acea6e1d468f8008d8a82b9, ASSIGN to state=OFFLINE, location=null, table=test, region=7830dc6b1acea6e1d468f8008d8a82b9 to restore RIT 2024-11-08T05:13:30,230 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=64ddf686fed30cdb23b2e874efec598d, ASSIGN to state=OFFLINE, location=null, table=test, region=64ddf686fed30cdb23b2e874efec598d to restore RIT 2024-11-08T05:13:30,231 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731042840231 2024-11-08T05:13:30,231 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-08T05:13:30,231 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure e61dc9e059921fddc5ad7f6f7971acc6, server=d1ba675678c9,41485,1731042802970:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d1ba675678c9,46753,1731042802851, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T05:13:30,233 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-08T05:13:30,234 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-08T05:13:30,234 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=72, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 868527ec322eba1d7cbd6371701f7ad2, server=d1ba675678c9,41485,1731042802970 2024-11-08T05:13:30,234 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T05:13:30,234 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=72, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 868527ec322eba1d7cbd6371701f7ad2, server=d1ba675678c9,41485,1731042802970 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T05:13:30,239 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6240e38715889f3bb77bb89478c81a6c, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,239 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eeaad0e7e7aa66f0b51c273c0aaf953c, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,239 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=68bbd0353f97ee12228f7798b0b987ea, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,239 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=64ddf686fed30cdb23b2e874efec598d, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,240 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d7680a9929f15640ca61551f186f33c1, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,240 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=65d5312dbd2f96bf3b16bdfd7504dff6, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,240 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9b3274d80dcb7deed617c19ac1de4a68, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,240 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2d23e3e11ecc480e314e989d1bb8d0c3, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,241 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fe1545b16a81758dcb9430dbf98c4ab9, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,241 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a12ea80f78b339cfbfc0e18b5338df0c, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,241 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e1c63297d3a8aea2fdbd1842301c811, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,241 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=40440b5f39bb5421276aa0b156481a9e, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,241 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=27c9aaf34aaf98cd5bc8030630ae7fe3, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,242 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=868527ec322eba1d7cbd6371701f7ad2, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,242 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d951febea78018bb9f10f07d8788575d, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,242 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b0ebd9a9794acc56e430b36d1f17621c, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,242 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7830dc6b1acea6e1d468f8008d8a82b9, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,242 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12f1148a7b8db3e4f515337b8b7746fa, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,242 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e26dcb2339b364f3699d110c15cd4bb3, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,243 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=445b14856951586918c91d4ba342f1ff, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,243 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=14efcc1b9822cc74f4d4a96648555167, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,243 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7c8f72849b2e1719ed903afc73e0571a, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,243 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=81806cb184c70c9dc0df408694092d85, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,243 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=87b3455cffd1f4e06e5901b3bcbed9dc, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,244 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=de358f808110a48fa32e0533fefdc95c, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,244 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e61dc9e059921fddc5ad7f6f7971acc6, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,244 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-08T05:13:30,244 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731042870244 2024-11-08T05:13:30,244 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731042930244 2024-11-08T05:13:30,244 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 13 msec 2024-11-08T05:13:30,244 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-08T05:13:30,244 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T05:13:30,244 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-08T05:13:30,245 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-08T05:13:30,245 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-08T05:13:30,245 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-08T05:13:30,245 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-08T05:13:30,245 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-08T05:13:30,246 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-08T05:13:30,250 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-08T05:13:30,250 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T05:13:30,250 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T05:13:30,250 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T05:13:30,250 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T05:13:30,250 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T05:13:30,250 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T05:13:30,250 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T05:13:30,250 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T05:13:30,250 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T05:13:30,250 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T05:13:30,250 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T05:13:30,250 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T05:13:30,250 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T05:13:30,250 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T05:13:30,250 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T05:13:30,251 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-08T05:13:30,252 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-08T05:13:30,255 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure e61dc9e059921fddc5ad7f6f7971acc6, server=d1ba675678c9,41485,1731042802970:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d1ba675678c9,46753,1731042802851, splitWal=true, meta=true 2024-11-08T05:13:30,255 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure d1ba675678c9%2C46753%2C1731042802851.meta.1731042805109.meta, worker=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:30,255 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d1ba675678c9%2C46753%2C1731042802851.meta.1731042805109.meta, worker=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:30,255 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T05:13:30,255 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d1ba675678c9,41485,1731042802970 2024-11-08T05:13:30,256 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure d1ba675678c9%2C46753%2C1731042802851.1731042804603, worker=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:30,256 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d1ba675678c9%2C46753%2C1731042802851.1731042804603, worker=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:30,256 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e61dc9e059921fddc5ad7f6f7971acc6, ASSIGN 2024-11-08T05:13:30,256 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=81806cb184c70c9dc0df408694092d85, ASSIGN 2024-11-08T05:13:30,256 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=868527ec322eba1d7cbd6371701f7ad2, ASSIGN 2024-11-08T05:13:30,256 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a12ea80f78b339cfbfc0e18b5338df0c, ASSIGN 2024-11-08T05:13:30,256 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=445b14856951586918c91d4ba342f1ff, ASSIGN 2024-11-08T05:13:30,256 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9b3274d80dcb7deed617c19ac1de4a68, ASSIGN 2024-11-08T05:13:30,256 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7830dc6b1acea6e1d468f8008d8a82b9, ASSIGN 2024-11-08T05:13:30,256 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=64ddf686fed30cdb23b2e874efec598d, ASSIGN 2024-11-08T05:13:30,256 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 868527ec322eba1d7cbd6371701f7ad2, server=d1ba675678c9,41485,1731042802970 2024-11-08T05:13:30,256 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure e61dc9e059921fddc5ad7f6f7971acc6, server=d1ba675678c9,41485,1731042802970:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d1ba675678c9,46753,1731042802851, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:30,256 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure d1ba675678c9%2C46753%2C1731042802851.meta.1731042805109.meta, worker=d1ba675678c9,39291,1731042803020 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,256 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d1ba675678c9%2C46753%2C1731042802851.meta.1731042805109.meta, worker=d1ba675678c9,39291,1731042803020 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,256 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,256 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d1ba675678c9,41485,1731042802970 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,256 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure d1ba675678c9%2C46753%2C1731042802851.1731042804603, worker=d1ba675678c9,39291,1731042803020 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,257 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d1ba675678c9%2C46753%2C1731042802851.1731042804603, worker=d1ba675678c9,39291,1731042803020 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,257 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e61dc9e059921fddc5ad7f6f7971acc6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:30,257 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e61dc9e059921fddc5ad7f6f7971acc6, ASSIGN 2024-11-08T05:13:30,257 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=81806cb184c70c9dc0df408694092d85, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:30,257 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=81806cb184c70c9dc0df408694092d85, ASSIGN 2024-11-08T05:13:30,257 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=868527ec322eba1d7cbd6371701f7ad2, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:30,257 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=868527ec322eba1d7cbd6371701f7ad2, ASSIGN 2024-11-08T05:13:30,257 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a12ea80f78b339cfbfc0e18b5338df0c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:30,257 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a12ea80f78b339cfbfc0e18b5338df0c, ASSIGN 2024-11-08T05:13:30,257 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=445b14856951586918c91d4ba342f1ff, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:30,257 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=445b14856951586918c91d4ba342f1ff, ASSIGN 2024-11-08T05:13:30,257 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9b3274d80dcb7deed617c19ac1de4a68, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:30,257 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9b3274d80dcb7deed617c19ac1de4a68, ASSIGN 2024-11-08T05:13:30,257 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7830dc6b1acea6e1d468f8008d8a82b9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:30,257 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7830dc6b1acea6e1d468f8008d8a82b9, ASSIGN 2024-11-08T05:13:30,257 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=64ddf686fed30cdb23b2e874efec598d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:30,257 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=64ddf686fed30cdb23b2e874efec598d, ASSIGN 2024-11-08T05:13:30,257 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 868527ec322eba1d7cbd6371701f7ad2, server=d1ba675678c9,41485,1731042802970 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,258 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 11 msec 2024-11-08T05:13:30,259 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=61 2024-11-08T05:13:30,259 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=d1ba675678c9,41485,1731042802970, table=hbase:meta, region=1588230740 2024-11-08T05:13:30,259 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d1ba675678c9,41485,1731042802970, state=OPEN 2024-11-08T05:13:30,271 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41485-0x10118dcefab0002, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T05:13:30,271 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39291-0x10118dcefab0003, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T05:13:30,271 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37587-0x10118dcefab0000, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T05:13:30,271 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T05:13:30,271 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T05:13:30,271 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T05:13:30,272 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e61dc9e059921fddc5ad7f6f7971acc6, ASSIGN to state=OFFLINE, location=null, table=test, region=e61dc9e059921fddc5ad7f6f7971acc6 to restore RIT 2024-11-08T05:13:30,272 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=81806cb184c70c9dc0df408694092d85, ASSIGN to state=OFFLINE, location=null, table=test, region=81806cb184c70c9dc0df408694092d85 to restore RIT 2024-11-08T05:13:30,272 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=868527ec322eba1d7cbd6371701f7ad2, ASSIGN to state=OFFLINE, location=null, table=test, region=868527ec322eba1d7cbd6371701f7ad2 to restore RIT 2024-11-08T05:13:30,272 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a12ea80f78b339cfbfc0e18b5338df0c, ASSIGN to state=OFFLINE, location=null, table=test, region=a12ea80f78b339cfbfc0e18b5338df0c to restore RIT 2024-11-08T05:13:30,272 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=445b14856951586918c91d4ba342f1ff, ASSIGN to state=OFFLINE, location=null, table=test, region=445b14856951586918c91d4ba342f1ff to restore RIT 2024-11-08T05:13:30,273 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9b3274d80dcb7deed617c19ac1de4a68, ASSIGN to state=OFFLINE, location=null, table=test, region=9b3274d80dcb7deed617c19ac1de4a68 to restore RIT 2024-11-08T05:13:30,273 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7830dc6b1acea6e1d468f8008d8a82b9, ASSIGN to state=OFFLINE, location=null, table=test, region=7830dc6b1acea6e1d468f8008d8a82b9 to restore RIT 2024-11-08T05:13:30,273 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=64ddf686fed30cdb23b2e874efec598d, ASSIGN to state=OFFLINE, location=null, table=test, region=64ddf686fed30cdb23b2e874efec598d to restore RIT 2024-11-08T05:13:30,273 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731042840273 2024-11-08T05:13:30,273 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-08T05:13:30,273 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure e61dc9e059921fddc5ad7f6f7971acc6, server=d1ba675678c9,41485,1731042802970:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d1ba675678c9,46753,1731042802851, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T05:13:30,275 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-08T05:13:30,277 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-08T05:13:30,277 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=64ddf686fed30cdb23b2e874efec598d, ASSIGN 2024-11-08T05:13:30,277 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T05:13:30,277 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=64ddf686fed30cdb23b2e874efec598d, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T05:13:30,281 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6240e38715889f3bb77bb89478c81a6c, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,281 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eeaad0e7e7aa66f0b51c273c0aaf953c, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,281 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=68bbd0353f97ee12228f7798b0b987ea, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,281 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=64ddf686fed30cdb23b2e874efec598d, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,281 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d7680a9929f15640ca61551f186f33c1, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,282 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=65d5312dbd2f96bf3b16bdfd7504dff6, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,282 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9b3274d80dcb7deed617c19ac1de4a68, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,282 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2d23e3e11ecc480e314e989d1bb8d0c3, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,282 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fe1545b16a81758dcb9430dbf98c4ab9, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,282 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a12ea80f78b339cfbfc0e18b5338df0c, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,283 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e1c63297d3a8aea2fdbd1842301c811, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,283 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=40440b5f39bb5421276aa0b156481a9e, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,283 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=27c9aaf34aaf98cd5bc8030630ae7fe3, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,283 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=868527ec322eba1d7cbd6371701f7ad2, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,284 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d951febea78018bb9f10f07d8788575d, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,284 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b0ebd9a9794acc56e430b36d1f17621c, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,284 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7830dc6b1acea6e1d468f8008d8a82b9, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,284 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12f1148a7b8db3e4f515337b8b7746fa, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,284 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e26dcb2339b364f3699d110c15cd4bb3, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,284 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=445b14856951586918c91d4ba342f1ff, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,285 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=14efcc1b9822cc74f4d4a96648555167, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,285 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7c8f72849b2e1719ed903afc73e0571a, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,285 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=81806cb184c70c9dc0df408694092d85, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,285 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=87b3455cffd1f4e06e5901b3bcbed9dc, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,285 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=de358f808110a48fa32e0533fefdc95c, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,285 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e61dc9e059921fddc5ad7f6f7971acc6, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,285 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-08T05:13:30,285 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731042870285 2024-11-08T05:13:30,286 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731042930285 2024-11-08T05:13:30,286 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 12 msec 2024-11-08T05:13:30,286 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-08T05:13:30,286 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T05:13:30,286 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-08T05:13:30,286 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-08T05:13:30,286 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-08T05:13:30,286 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-08T05:13:30,286 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-08T05:13:30,287 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-08T05:13:30,287 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37587 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T05:13:30,288 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-08T05:13:30,288 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37587 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T05:13:30,290 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-08T05:13:30,291 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T05:13:30,291 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T05:13:30,291 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T05:13:30,291 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T05:13:30,291 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T05:13:30,291 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T05:13:30,291 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T05:13:30,291 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T05:13:30,291 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T05:13:30,291 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T05:13:30,291 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T05:13:30,291 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T05:13:30,291 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T05:13:30,291 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T05:13:30,292 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-08T05:13:30,293 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-08T05:13:30,295 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure e61dc9e059921fddc5ad7f6f7971acc6, server=d1ba675678c9,41485,1731042802970:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d1ba675678c9,46753,1731042802851, splitWal=true, meta=true 2024-11-08T05:13:30,295 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure d1ba675678c9%2C46753%2C1731042802851.meta.1731042805109.meta, worker=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:30,295 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d1ba675678c9%2C46753%2C1731042802851.meta.1731042805109.meta, worker=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:30,295 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T05:13:30,295 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d1ba675678c9,41485,1731042802970 2024-11-08T05:13:30,295 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure d1ba675678c9%2C46753%2C1731042802851.1731042804603, worker=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:30,295 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d1ba675678c9%2C46753%2C1731042802851.1731042804603, worker=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:30,295 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e61dc9e059921fddc5ad7f6f7971acc6, ASSIGN 2024-11-08T05:13:30,295 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=81806cb184c70c9dc0df408694092d85, ASSIGN 2024-11-08T05:13:30,295 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=868527ec322eba1d7cbd6371701f7ad2, ASSIGN 2024-11-08T05:13:30,295 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a12ea80f78b339cfbfc0e18b5338df0c, ASSIGN 2024-11-08T05:13:30,296 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=445b14856951586918c91d4ba342f1ff, ASSIGN 2024-11-08T05:13:30,296 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9b3274d80dcb7deed617c19ac1de4a68, ASSIGN 2024-11-08T05:13:30,296 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7830dc6b1acea6e1d468f8008d8a82b9, ASSIGN 2024-11-08T05:13:30,296 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=64ddf686fed30cdb23b2e874efec598d, ASSIGN 2024-11-08T05:13:30,296 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure e61dc9e059921fddc5ad7f6f7971acc6, server=d1ba675678c9,41485,1731042802970:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d1ba675678c9,46753,1731042802851, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:30,296 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure d1ba675678c9%2C46753%2C1731042802851.meta.1731042805109.meta, worker=d1ba675678c9,39291,1731042803020 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,296 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d1ba675678c9%2C46753%2C1731042802851.meta.1731042805109.meta, worker=d1ba675678c9,39291,1731042803020 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,296 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,296 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d1ba675678c9,41485,1731042802970 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,296 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure d1ba675678c9%2C46753%2C1731042802851.1731042804603, worker=d1ba675678c9,39291,1731042803020 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,296 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d1ba675678c9%2C46753%2C1731042802851.1731042804603, worker=d1ba675678c9,39291,1731042803020 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,296 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e61dc9e059921fddc5ad7f6f7971acc6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:30,296 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e61dc9e059921fddc5ad7f6f7971acc6, ASSIGN 2024-11-08T05:13:30,296 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=81806cb184c70c9dc0df408694092d85, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:30,296 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=81806cb184c70c9dc0df408694092d85, ASSIGN 2024-11-08T05:13:30,296 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=868527ec322eba1d7cbd6371701f7ad2, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:30,296 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=868527ec322eba1d7cbd6371701f7ad2, ASSIGN 2024-11-08T05:13:30,296 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a12ea80f78b339cfbfc0e18b5338df0c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:30,296 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a12ea80f78b339cfbfc0e18b5338df0c, ASSIGN 2024-11-08T05:13:30,297 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=445b14856951586918c91d4ba342f1ff, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:30,297 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=445b14856951586918c91d4ba342f1ff, ASSIGN 2024-11-08T05:13:30,297 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9b3274d80dcb7deed617c19ac1de4a68, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:30,297 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9b3274d80dcb7deed617c19ac1de4a68, ASSIGN 2024-11-08T05:13:30,297 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7830dc6b1acea6e1d468f8008d8a82b9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:30,297 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7830dc6b1acea6e1d468f8008d8a82b9, ASSIGN 2024-11-08T05:13:30,297 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=64ddf686fed30cdb23b2e874efec598d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:30,297 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=64ddf686fed30cdb23b2e874efec598d, ASSIGN 2024-11-08T05:13:30,297 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 9 msec 2024-11-08T05:13:30,298 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=61 2024-11-08T05:13:30,298 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=d1ba675678c9,41485,1731042802970, table=hbase:meta, region=1588230740 2024-11-08T05:13:30,298 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d1ba675678c9,41485,1731042802970, state=OPEN 2024-11-08T05:13:30,309 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39291-0x10118dcefab0003, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T05:13:30,309 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37587-0x10118dcefab0000, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T05:13:30,309 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41485-0x10118dcefab0002, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T05:13:30,310 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T05:13:30,310 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T05:13:30,310 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T05:13:30,310 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e61dc9e059921fddc5ad7f6f7971acc6, ASSIGN to state=OFFLINE, location=null, table=test, region=e61dc9e059921fddc5ad7f6f7971acc6 to restore RIT 2024-11-08T05:13:30,310 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=81806cb184c70c9dc0df408694092d85, ASSIGN to state=OFFLINE, location=null, table=test, region=81806cb184c70c9dc0df408694092d85 to restore RIT 2024-11-08T05:13:30,310 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=868527ec322eba1d7cbd6371701f7ad2, ASSIGN to state=OFFLINE, location=null, table=test, region=868527ec322eba1d7cbd6371701f7ad2 to restore RIT 2024-11-08T05:13:30,310 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a12ea80f78b339cfbfc0e18b5338df0c, ASSIGN to state=OFFLINE, location=null, table=test, region=a12ea80f78b339cfbfc0e18b5338df0c to restore RIT 2024-11-08T05:13:30,310 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=445b14856951586918c91d4ba342f1ff, ASSIGN to state=OFFLINE, location=null, table=test, region=445b14856951586918c91d4ba342f1ff to restore RIT 2024-11-08T05:13:30,310 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9b3274d80dcb7deed617c19ac1de4a68, ASSIGN to state=OFFLINE, location=null, table=test, region=9b3274d80dcb7deed617c19ac1de4a68 to restore RIT 2024-11-08T05:13:30,310 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7830dc6b1acea6e1d468f8008d8a82b9, ASSIGN to state=OFFLINE, location=null, table=test, region=7830dc6b1acea6e1d468f8008d8a82b9 to restore RIT 2024-11-08T05:13:30,310 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=64ddf686fed30cdb23b2e874efec598d, ASSIGN to state=OFFLINE, location=null, table=test, region=64ddf686fed30cdb23b2e874efec598d to restore RIT 2024-11-08T05:13:30,311 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731042840311 2024-11-08T05:13:30,311 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-08T05:13:30,311 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure e61dc9e059921fddc5ad7f6f7971acc6, server=d1ba675678c9,41485,1731042802970:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d1ba675678c9,46753,1731042802851, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T05:13:30,312 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-08T05:13:30,313 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-08T05:13:30,313 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7830dc6b1acea6e1d468f8008d8a82b9, ASSIGN 2024-11-08T05:13:30,313 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T05:13:30,314 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7830dc6b1acea6e1d468f8008d8a82b9, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T05:13:30,316 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6240e38715889f3bb77bb89478c81a6c, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,316 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eeaad0e7e7aa66f0b51c273c0aaf953c, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,317 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=68bbd0353f97ee12228f7798b0b987ea, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,317 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=64ddf686fed30cdb23b2e874efec598d, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,317 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d7680a9929f15640ca61551f186f33c1, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,317 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=65d5312dbd2f96bf3b16bdfd7504dff6, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,317 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9b3274d80dcb7deed617c19ac1de4a68, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,317 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2d23e3e11ecc480e314e989d1bb8d0c3, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,317 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fe1545b16a81758dcb9430dbf98c4ab9, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,317 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a12ea80f78b339cfbfc0e18b5338df0c, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,318 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e1c63297d3a8aea2fdbd1842301c811, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,318 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=40440b5f39bb5421276aa0b156481a9e, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,318 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=27c9aaf34aaf98cd5bc8030630ae7fe3, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,318 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=868527ec322eba1d7cbd6371701f7ad2, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,318 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d951febea78018bb9f10f07d8788575d, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,318 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b0ebd9a9794acc56e430b36d1f17621c, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,318 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7830dc6b1acea6e1d468f8008d8a82b9, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,318 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12f1148a7b8db3e4f515337b8b7746fa, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,319 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e26dcb2339b364f3699d110c15cd4bb3, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,319 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=445b14856951586918c91d4ba342f1ff, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,319 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=14efcc1b9822cc74f4d4a96648555167, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,319 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7c8f72849b2e1719ed903afc73e0571a, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,319 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=81806cb184c70c9dc0df408694092d85, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,319 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=87b3455cffd1f4e06e5901b3bcbed9dc, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,319 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=de358f808110a48fa32e0533fefdc95c, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,319 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e61dc9e059921fddc5ad7f6f7971acc6, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,319 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-08T05:13:30,319 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731042870319 2024-11-08T05:13:30,320 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731042930320 2024-11-08T05:13:30,320 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 8 msec 2024-11-08T05:13:30,320 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-08T05:13:30,320 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T05:13:30,320 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-08T05:13:30,320 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-08T05:13:30,320 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-08T05:13:30,320 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-08T05:13:30,320 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-08T05:13:30,320 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-08T05:13:30,321 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-08T05:13:30,323 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-08T05:13:30,323 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T05:13:30,323 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T05:13:30,323 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T05:13:30,323 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T05:13:30,324 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T05:13:30,324 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T05:13:30,324 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T05:13:30,324 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T05:13:30,324 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T05:13:30,324 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T05:13:30,324 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T05:13:30,324 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T05:13:30,324 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T05:13:30,324 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-08T05:13:30,325 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-08T05:13:30,326 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure e61dc9e059921fddc5ad7f6f7971acc6, server=d1ba675678c9,41485,1731042802970:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d1ba675678c9,46753,1731042802851, splitWal=true, meta=true 2024-11-08T05:13:30,326 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure d1ba675678c9%2C46753%2C1731042802851.meta.1731042805109.meta, worker=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:30,326 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d1ba675678c9%2C46753%2C1731042802851.meta.1731042805109.meta, worker=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:30,326 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T05:13:30,326 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d1ba675678c9,41485,1731042802970 2024-11-08T05:13:30,326 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure d1ba675678c9%2C46753%2C1731042802851.1731042804603, worker=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:30,326 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d1ba675678c9%2C46753%2C1731042802851.1731042804603, worker=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:30,326 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e61dc9e059921fddc5ad7f6f7971acc6, ASSIGN 2024-11-08T05:13:30,326 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=81806cb184c70c9dc0df408694092d85, ASSIGN 2024-11-08T05:13:30,326 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=868527ec322eba1d7cbd6371701f7ad2, ASSIGN 2024-11-08T05:13:30,326 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a12ea80f78b339cfbfc0e18b5338df0c, ASSIGN 2024-11-08T05:13:30,326 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=445b14856951586918c91d4ba342f1ff, ASSIGN 2024-11-08T05:13:30,327 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9b3274d80dcb7deed617c19ac1de4a68, ASSIGN 2024-11-08T05:13:30,327 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7830dc6b1acea6e1d468f8008d8a82b9, ASSIGN 2024-11-08T05:13:30,327 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure e61dc9e059921fddc5ad7f6f7971acc6, server=d1ba675678c9,41485,1731042802970:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d1ba675678c9,46753,1731042802851, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:30,327 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure d1ba675678c9%2C46753%2C1731042802851.meta.1731042805109.meta, worker=d1ba675678c9,39291,1731042803020 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,327 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d1ba675678c9%2C46753%2C1731042802851.meta.1731042805109.meta, worker=d1ba675678c9,39291,1731042803020 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,327 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,327 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d1ba675678c9,41485,1731042802970 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,327 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure d1ba675678c9%2C46753%2C1731042802851.1731042804603, worker=d1ba675678c9,39291,1731042803020 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,327 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d1ba675678c9%2C46753%2C1731042802851.1731042804603, worker=d1ba675678c9,39291,1731042803020 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,327 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e61dc9e059921fddc5ad7f6f7971acc6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:30,327 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e61dc9e059921fddc5ad7f6f7971acc6, ASSIGN 2024-11-08T05:13:30,327 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=81806cb184c70c9dc0df408694092d85, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:30,327 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=81806cb184c70c9dc0df408694092d85, ASSIGN 2024-11-08T05:13:30,327 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=868527ec322eba1d7cbd6371701f7ad2, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:30,327 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=868527ec322eba1d7cbd6371701f7ad2, ASSIGN 2024-11-08T05:13:30,327 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a12ea80f78b339cfbfc0e18b5338df0c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:30,327 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a12ea80f78b339cfbfc0e18b5338df0c, ASSIGN 2024-11-08T05:13:30,327 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=445b14856951586918c91d4ba342f1ff, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:30,327 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=445b14856951586918c91d4ba342f1ff, ASSIGN 2024-11-08T05:13:30,327 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9b3274d80dcb7deed617c19ac1de4a68, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:30,327 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9b3274d80dcb7deed617c19ac1de4a68, ASSIGN 2024-11-08T05:13:30,327 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7830dc6b1acea6e1d468f8008d8a82b9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:30,327 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7830dc6b1acea6e1d468f8008d8a82b9, ASSIGN 2024-11-08T05:13:30,328 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-08T05:13:30,328 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=61 2024-11-08T05:13:30,328 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=d1ba675678c9,41485,1731042802970, table=hbase:meta, region=1588230740 2024-11-08T05:13:30,329 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d1ba675678c9,41485,1731042802970, state=OPEN 2024-11-08T05:13:30,341 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39291-0x10118dcefab0003, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T05:13:30,341 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41485-0x10118dcefab0002, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T05:13:30,341 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37587-0x10118dcefab0000, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T05:13:30,341 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T05:13:30,341 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T05:13:30,341 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T05:13:30,342 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e61dc9e059921fddc5ad7f6f7971acc6, ASSIGN to state=OFFLINE, location=null, table=test, region=e61dc9e059921fddc5ad7f6f7971acc6 to restore RIT 2024-11-08T05:13:30,342 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=81806cb184c70c9dc0df408694092d85, ASSIGN to state=OFFLINE, location=null, table=test, region=81806cb184c70c9dc0df408694092d85 to restore RIT 2024-11-08T05:13:30,342 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=868527ec322eba1d7cbd6371701f7ad2, ASSIGN to state=OFFLINE, location=null, table=test, region=868527ec322eba1d7cbd6371701f7ad2 to restore RIT 2024-11-08T05:13:30,342 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a12ea80f78b339cfbfc0e18b5338df0c, ASSIGN to state=OFFLINE, location=null, table=test, region=a12ea80f78b339cfbfc0e18b5338df0c to restore RIT 2024-11-08T05:13:30,342 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=445b14856951586918c91d4ba342f1ff, ASSIGN to state=OFFLINE, location=null, table=test, region=445b14856951586918c91d4ba342f1ff to restore RIT 2024-11-08T05:13:30,342 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9b3274d80dcb7deed617c19ac1de4a68, ASSIGN to state=OFFLINE, location=null, table=test, region=9b3274d80dcb7deed617c19ac1de4a68 to restore RIT 2024-11-08T05:13:30,342 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7830dc6b1acea6e1d468f8008d8a82b9, ASSIGN to state=OFFLINE, location=null, table=test, region=7830dc6b1acea6e1d468f8008d8a82b9 to restore RIT 2024-11-08T05:13:30,343 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731042840343 2024-11-08T05:13:30,343 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-08T05:13:30,343 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure e61dc9e059921fddc5ad7f6f7971acc6, server=d1ba675678c9,41485,1731042802970:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d1ba675678c9,46753,1731042802851, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T05:13:30,344 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-08T05:13:30,345 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-08T05:13:30,345 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9b3274d80dcb7deed617c19ac1de4a68, ASSIGN 2024-11-08T05:13:30,345 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T05:13:30,345 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9b3274d80dcb7deed617c19ac1de4a68, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T05:13:30,348 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6240e38715889f3bb77bb89478c81a6c, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,349 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eeaad0e7e7aa66f0b51c273c0aaf953c, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,349 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=68bbd0353f97ee12228f7798b0b987ea, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,349 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=64ddf686fed30cdb23b2e874efec598d, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,349 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d7680a9929f15640ca61551f186f33c1, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,349 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=65d5312dbd2f96bf3b16bdfd7504dff6, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,349 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9b3274d80dcb7deed617c19ac1de4a68, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,349 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2d23e3e11ecc480e314e989d1bb8d0c3, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,350 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fe1545b16a81758dcb9430dbf98c4ab9, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,350 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a12ea80f78b339cfbfc0e18b5338df0c, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,350 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e1c63297d3a8aea2fdbd1842301c811, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,350 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=40440b5f39bb5421276aa0b156481a9e, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,350 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=27c9aaf34aaf98cd5bc8030630ae7fe3, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,350 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=868527ec322eba1d7cbd6371701f7ad2, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,350 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d951febea78018bb9f10f07d8788575d, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,351 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b0ebd9a9794acc56e430b36d1f17621c, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,351 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7830dc6b1acea6e1d468f8008d8a82b9, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,351 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12f1148a7b8db3e4f515337b8b7746fa, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,351 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e26dcb2339b364f3699d110c15cd4bb3, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,351 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=445b14856951586918c91d4ba342f1ff, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,351 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=14efcc1b9822cc74f4d4a96648555167, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,351 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7c8f72849b2e1719ed903afc73e0571a, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,352 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=81806cb184c70c9dc0df408694092d85, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,352 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=87b3455cffd1f4e06e5901b3bcbed9dc, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,352 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=de358f808110a48fa32e0533fefdc95c, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,352 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e61dc9e059921fddc5ad7f6f7971acc6, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,352 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-08T05:13:30,352 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731042870352 2024-11-08T05:13:30,352 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731042930352 2024-11-08T05:13:30,352 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 9 msec 2024-11-08T05:13:30,352 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-08T05:13:30,352 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T05:13:30,352 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-08T05:13:30,353 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-08T05:13:30,353 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-08T05:13:30,353 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-08T05:13:30,353 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-08T05:13:30,353 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-08T05:13:30,354 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-08T05:13:30,356 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-08T05:13:30,356 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T05:13:30,356 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T05:13:30,357 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T05:13:30,357 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T05:13:30,357 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T05:13:30,357 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T05:13:30,357 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T05:13:30,357 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T05:13:30,357 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T05:13:30,357 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T05:13:30,357 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T05:13:30,357 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T05:13:30,357 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-08T05:13:30,358 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-08T05:13:30,359 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure e61dc9e059921fddc5ad7f6f7971acc6, server=d1ba675678c9,41485,1731042802970:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d1ba675678c9,46753,1731042802851, splitWal=true, meta=true 2024-11-08T05:13:30,359 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure d1ba675678c9%2C46753%2C1731042802851.meta.1731042805109.meta, worker=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:30,359 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d1ba675678c9%2C46753%2C1731042802851.meta.1731042805109.meta, worker=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:30,359 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T05:13:30,359 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d1ba675678c9,41485,1731042802970 2024-11-08T05:13:30,359 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure d1ba675678c9%2C46753%2C1731042802851.1731042804603, worker=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:30,359 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d1ba675678c9%2C46753%2C1731042802851.1731042804603, worker=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:30,359 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e61dc9e059921fddc5ad7f6f7971acc6, ASSIGN 2024-11-08T05:13:30,359 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=81806cb184c70c9dc0df408694092d85, ASSIGN 2024-11-08T05:13:30,359 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=868527ec322eba1d7cbd6371701f7ad2, ASSIGN 2024-11-08T05:13:30,360 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a12ea80f78b339cfbfc0e18b5338df0c, ASSIGN 2024-11-08T05:13:30,360 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=445b14856951586918c91d4ba342f1ff, ASSIGN 2024-11-08T05:13:30,360 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9b3274d80dcb7deed617c19ac1de4a68, ASSIGN 2024-11-08T05:13:30,360 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure e61dc9e059921fddc5ad7f6f7971acc6, server=d1ba675678c9,41485,1731042802970:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d1ba675678c9,46753,1731042802851, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:30,360 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure d1ba675678c9%2C46753%2C1731042802851.meta.1731042805109.meta, worker=d1ba675678c9,39291,1731042803020 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,360 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d1ba675678c9%2C46753%2C1731042802851.meta.1731042805109.meta, worker=d1ba675678c9,39291,1731042803020 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,360 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,360 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d1ba675678c9,41485,1731042802970 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,360 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure d1ba675678c9%2C46753%2C1731042802851.1731042804603, worker=d1ba675678c9,39291,1731042803020 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,360 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d1ba675678c9%2C46753%2C1731042802851.1731042804603, worker=d1ba675678c9,39291,1731042803020 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,360 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e61dc9e059921fddc5ad7f6f7971acc6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:30,360 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e61dc9e059921fddc5ad7f6f7971acc6, ASSIGN 2024-11-08T05:13:30,360 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=81806cb184c70c9dc0df408694092d85, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:30,360 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=81806cb184c70c9dc0df408694092d85, ASSIGN 2024-11-08T05:13:30,360 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=868527ec322eba1d7cbd6371701f7ad2, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:30,360 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=868527ec322eba1d7cbd6371701f7ad2, ASSIGN 2024-11-08T05:13:30,360 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a12ea80f78b339cfbfc0e18b5338df0c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:30,360 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a12ea80f78b339cfbfc0e18b5338df0c, ASSIGN 2024-11-08T05:13:30,360 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=445b14856951586918c91d4ba342f1ff, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:30,361 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=445b14856951586918c91d4ba342f1ff, ASSIGN 2024-11-08T05:13:30,361 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9b3274d80dcb7deed617c19ac1de4a68, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:30,361 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9b3274d80dcb7deed617c19ac1de4a68, ASSIGN 2024-11-08T05:13:30,361 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-08T05:13:30,362 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=61 2024-11-08T05:13:30,362 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=d1ba675678c9,41485,1731042802970, table=hbase:meta, region=1588230740 2024-11-08T05:13:30,362 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d1ba675678c9,41485,1731042802970, state=OPEN 2024-11-08T05:13:30,373 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39291-0x10118dcefab0003, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T05:13:30,373 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41485-0x10118dcefab0002, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T05:13:30,373 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37587-0x10118dcefab0000, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T05:13:30,373 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T05:13:30,373 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T05:13:30,373 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T05:13:30,374 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e61dc9e059921fddc5ad7f6f7971acc6, ASSIGN to state=OFFLINE, location=null, table=test, region=e61dc9e059921fddc5ad7f6f7971acc6 to restore RIT 2024-11-08T05:13:30,374 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=81806cb184c70c9dc0df408694092d85, ASSIGN to state=OFFLINE, location=null, table=test, region=81806cb184c70c9dc0df408694092d85 to restore RIT 2024-11-08T05:13:30,374 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=868527ec322eba1d7cbd6371701f7ad2, ASSIGN to state=OFFLINE, location=null, table=test, region=868527ec322eba1d7cbd6371701f7ad2 to restore RIT 2024-11-08T05:13:30,374 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a12ea80f78b339cfbfc0e18b5338df0c, ASSIGN to state=OFFLINE, location=null, table=test, region=a12ea80f78b339cfbfc0e18b5338df0c to restore RIT 2024-11-08T05:13:30,374 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=445b14856951586918c91d4ba342f1ff, ASSIGN to state=OFFLINE, location=null, table=test, region=445b14856951586918c91d4ba342f1ff to restore RIT 2024-11-08T05:13:30,374 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9b3274d80dcb7deed617c19ac1de4a68, ASSIGN to state=OFFLINE, location=null, table=test, region=9b3274d80dcb7deed617c19ac1de4a68 to restore RIT 2024-11-08T05:13:30,374 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731042840374 2024-11-08T05:13:30,374 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-08T05:13:30,374 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure e61dc9e059921fddc5ad7f6f7971acc6, server=d1ba675678c9,41485,1731042802970:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d1ba675678c9,46753,1731042802851, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T05:13:30,376 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-08T05:13:30,377 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-08T05:13:30,377 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=445b14856951586918c91d4ba342f1ff, ASSIGN 2024-11-08T05:13:30,377 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T05:13:30,377 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=445b14856951586918c91d4ba342f1ff, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T05:13:30,380 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6240e38715889f3bb77bb89478c81a6c, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,381 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eeaad0e7e7aa66f0b51c273c0aaf953c, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,381 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=68bbd0353f97ee12228f7798b0b987ea, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,381 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=64ddf686fed30cdb23b2e874efec598d, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,381 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d7680a9929f15640ca61551f186f33c1, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,381 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=65d5312dbd2f96bf3b16bdfd7504dff6, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,381 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9b3274d80dcb7deed617c19ac1de4a68, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,382 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2d23e3e11ecc480e314e989d1bb8d0c3, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,382 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fe1545b16a81758dcb9430dbf98c4ab9, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,382 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a12ea80f78b339cfbfc0e18b5338df0c, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,382 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e1c63297d3a8aea2fdbd1842301c811, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,382 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=40440b5f39bb5421276aa0b156481a9e, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,382 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=27c9aaf34aaf98cd5bc8030630ae7fe3, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,382 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=868527ec322eba1d7cbd6371701f7ad2, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,382 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d951febea78018bb9f10f07d8788575d, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,383 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b0ebd9a9794acc56e430b36d1f17621c, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,383 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7830dc6b1acea6e1d468f8008d8a82b9, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,383 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12f1148a7b8db3e4f515337b8b7746fa, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,383 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e26dcb2339b364f3699d110c15cd4bb3, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,383 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=445b14856951586918c91d4ba342f1ff, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,383 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=14efcc1b9822cc74f4d4a96648555167, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,383 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7c8f72849b2e1719ed903afc73e0571a, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,384 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=81806cb184c70c9dc0df408694092d85, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,384 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=87b3455cffd1f4e06e5901b3bcbed9dc, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,384 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=de358f808110a48fa32e0533fefdc95c, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,384 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e61dc9e059921fddc5ad7f6f7971acc6, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,384 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-08T05:13:30,384 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731042870384 2024-11-08T05:13:30,384 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731042930384 2024-11-08T05:13:30,384 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 9 msec 2024-11-08T05:13:30,384 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-08T05:13:30,384 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T05:13:30,385 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-08T05:13:30,385 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-08T05:13:30,385 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-08T05:13:30,385 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-08T05:13:30,385 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-08T05:13:30,385 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-08T05:13:30,386 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-08T05:13:30,388 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-08T05:13:30,389 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T05:13:30,389 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T05:13:30,389 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T05:13:30,389 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T05:13:30,389 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T05:13:30,389 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T05:13:30,389 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T05:13:30,389 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T05:13:30,389 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T05:13:30,389 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T05:13:30,389 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T05:13:30,389 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-08T05:13:30,390 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-08T05:13:30,391 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37587 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T05:13:30,391 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure e61dc9e059921fddc5ad7f6f7971acc6, server=d1ba675678c9,41485,1731042802970:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d1ba675678c9,46753,1731042802851, splitWal=true, meta=true 2024-11-08T05:13:30,391 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure d1ba675678c9%2C46753%2C1731042802851.meta.1731042805109.meta, worker=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:30,391 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d1ba675678c9%2C46753%2C1731042802851.meta.1731042805109.meta, worker=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:30,391 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T05:13:30,391 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d1ba675678c9,41485,1731042802970 2024-11-08T05:13:30,391 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure d1ba675678c9%2C46753%2C1731042802851.1731042804603, worker=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:30,392 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d1ba675678c9%2C46753%2C1731042802851.1731042804603, worker=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:30,391 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37587 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T05:13:30,392 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e61dc9e059921fddc5ad7f6f7971acc6, ASSIGN 2024-11-08T05:13:30,392 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=81806cb184c70c9dc0df408694092d85, ASSIGN 2024-11-08T05:13:30,392 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=868527ec322eba1d7cbd6371701f7ad2, ASSIGN 2024-11-08T05:13:30,392 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a12ea80f78b339cfbfc0e18b5338df0c, ASSIGN 2024-11-08T05:13:30,392 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=445b14856951586918c91d4ba342f1ff, ASSIGN 2024-11-08T05:13:30,392 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure e61dc9e059921fddc5ad7f6f7971acc6, server=d1ba675678c9,41485,1731042802970:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d1ba675678c9,46753,1731042802851, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:30,392 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure d1ba675678c9%2C46753%2C1731042802851.meta.1731042805109.meta, worker=d1ba675678c9,39291,1731042803020 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,392 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d1ba675678c9%2C46753%2C1731042802851.meta.1731042805109.meta, worker=d1ba675678c9,39291,1731042803020 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,392 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,392 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d1ba675678c9,41485,1731042802970 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,392 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure d1ba675678c9%2C46753%2C1731042802851.1731042804603, worker=d1ba675678c9,39291,1731042803020 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,392 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d1ba675678c9%2C46753%2C1731042802851.1731042804603, worker=d1ba675678c9,39291,1731042803020 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,392 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e61dc9e059921fddc5ad7f6f7971acc6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:30,392 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e61dc9e059921fddc5ad7f6f7971acc6, ASSIGN 2024-11-08T05:13:30,392 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=81806cb184c70c9dc0df408694092d85, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:30,393 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=81806cb184c70c9dc0df408694092d85, ASSIGN 2024-11-08T05:13:30,393 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=868527ec322eba1d7cbd6371701f7ad2, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:30,393 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=868527ec322eba1d7cbd6371701f7ad2, ASSIGN 2024-11-08T05:13:30,393 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a12ea80f78b339cfbfc0e18b5338df0c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:30,393 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a12ea80f78b339cfbfc0e18b5338df0c, ASSIGN 2024-11-08T05:13:30,393 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=445b14856951586918c91d4ba342f1ff, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:30,393 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=445b14856951586918c91d4ba342f1ff, ASSIGN 2024-11-08T05:13:30,393 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-08T05:13:30,394 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=61 2024-11-08T05:13:30,394 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=d1ba675678c9,41485,1731042802970, table=hbase:meta, region=1588230740 2024-11-08T05:13:30,394 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d1ba675678c9,41485,1731042802970, state=OPEN 2024-11-08T05:13:30,404 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39291-0x10118dcefab0003, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T05:13:30,404 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41485-0x10118dcefab0002, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T05:13:30,404 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37587-0x10118dcefab0000, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T05:13:30,404 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T05:13:30,404 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T05:13:30,404 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T05:13:30,405 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e61dc9e059921fddc5ad7f6f7971acc6, ASSIGN to state=OFFLINE, location=null, table=test, region=e61dc9e059921fddc5ad7f6f7971acc6 to restore RIT 2024-11-08T05:13:30,405 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=81806cb184c70c9dc0df408694092d85, ASSIGN to state=OFFLINE, location=null, table=test, region=81806cb184c70c9dc0df408694092d85 to restore RIT 2024-11-08T05:13:30,405 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=868527ec322eba1d7cbd6371701f7ad2, ASSIGN to state=OFFLINE, location=null, table=test, region=868527ec322eba1d7cbd6371701f7ad2 to restore RIT 2024-11-08T05:13:30,405 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a12ea80f78b339cfbfc0e18b5338df0c, ASSIGN to state=OFFLINE, location=null, table=test, region=a12ea80f78b339cfbfc0e18b5338df0c to restore RIT 2024-11-08T05:13:30,405 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=445b14856951586918c91d4ba342f1ff, ASSIGN to state=OFFLINE, location=null, table=test, region=445b14856951586918c91d4ba342f1ff to restore RIT 2024-11-08T05:13:30,406 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731042840406 2024-11-08T05:13:30,406 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-08T05:13:30,406 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure e61dc9e059921fddc5ad7f6f7971acc6, server=d1ba675678c9,41485,1731042802970:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d1ba675678c9,46753,1731042802851, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T05:13:30,407 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-08T05:13:30,408 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-08T05:13:30,408 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a12ea80f78b339cfbfc0e18b5338df0c, ASSIGN 2024-11-08T05:13:30,408 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T05:13:30,408 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a12ea80f78b339cfbfc0e18b5338df0c, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T05:13:30,412 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6240e38715889f3bb77bb89478c81a6c, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,412 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eeaad0e7e7aa66f0b51c273c0aaf953c, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,412 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=68bbd0353f97ee12228f7798b0b987ea, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,412 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=64ddf686fed30cdb23b2e874efec598d, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,412 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d7680a9929f15640ca61551f186f33c1, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,413 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=65d5312dbd2f96bf3b16bdfd7504dff6, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,413 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9b3274d80dcb7deed617c19ac1de4a68, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,413 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2d23e3e11ecc480e314e989d1bb8d0c3, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,413 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fe1545b16a81758dcb9430dbf98c4ab9, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,413 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a12ea80f78b339cfbfc0e18b5338df0c, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,413 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e1c63297d3a8aea2fdbd1842301c811, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,413 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=40440b5f39bb5421276aa0b156481a9e, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,414 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=27c9aaf34aaf98cd5bc8030630ae7fe3, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,414 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=868527ec322eba1d7cbd6371701f7ad2, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,414 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d951febea78018bb9f10f07d8788575d, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,414 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b0ebd9a9794acc56e430b36d1f17621c, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,414 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7830dc6b1acea6e1d468f8008d8a82b9, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,414 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12f1148a7b8db3e4f515337b8b7746fa, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,415 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e26dcb2339b364f3699d110c15cd4bb3, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,415 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=445b14856951586918c91d4ba342f1ff, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,415 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=14efcc1b9822cc74f4d4a96648555167, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,415 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7c8f72849b2e1719ed903afc73e0571a, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,415 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=81806cb184c70c9dc0df408694092d85, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,415 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=87b3455cffd1f4e06e5901b3bcbed9dc, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,416 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=de358f808110a48fa32e0533fefdc95c, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,416 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e61dc9e059921fddc5ad7f6f7971acc6, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,416 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-08T05:13:30,416 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731042870416 2024-11-08T05:13:30,416 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731042930416 2024-11-08T05:13:30,416 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 10 msec 2024-11-08T05:13:30,416 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-08T05:13:30,416 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T05:13:30,416 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-08T05:13:30,416 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-08T05:13:30,417 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-08T05:13:30,417 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-08T05:13:30,417 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-08T05:13:30,417 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-08T05:13:30,418 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-08T05:13:30,420 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-08T05:13:30,421 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T05:13:30,421 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T05:13:30,421 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T05:13:30,421 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T05:13:30,421 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T05:13:30,421 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T05:13:30,421 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T05:13:30,421 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T05:13:30,421 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T05:13:30,421 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T05:13:30,421 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-08T05:13:30,422 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-08T05:13:30,423 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure e61dc9e059921fddc5ad7f6f7971acc6, server=d1ba675678c9,41485,1731042802970:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d1ba675678c9,46753,1731042802851, splitWal=true, meta=true 2024-11-08T05:13:30,424 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure d1ba675678c9%2C46753%2C1731042802851.meta.1731042805109.meta, worker=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:30,424 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d1ba675678c9%2C46753%2C1731042802851.meta.1731042805109.meta, worker=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:30,424 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T05:13:30,424 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d1ba675678c9,41485,1731042802970 2024-11-08T05:13:30,424 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure d1ba675678c9%2C46753%2C1731042802851.1731042804603, worker=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:30,424 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d1ba675678c9%2C46753%2C1731042802851.1731042804603, worker=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:30,424 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e61dc9e059921fddc5ad7f6f7971acc6, ASSIGN 2024-11-08T05:13:30,424 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=81806cb184c70c9dc0df408694092d85, ASSIGN 2024-11-08T05:13:30,424 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=868527ec322eba1d7cbd6371701f7ad2, ASSIGN 2024-11-08T05:13:30,424 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a12ea80f78b339cfbfc0e18b5338df0c, ASSIGN 2024-11-08T05:13:30,424 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure e61dc9e059921fddc5ad7f6f7971acc6, server=d1ba675678c9,41485,1731042802970:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d1ba675678c9,46753,1731042802851, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:30,424 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure d1ba675678c9%2C46753%2C1731042802851.meta.1731042805109.meta, worker=d1ba675678c9,39291,1731042803020 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,424 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d1ba675678c9%2C46753%2C1731042802851.meta.1731042805109.meta, worker=d1ba675678c9,39291,1731042803020 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,425 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,425 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d1ba675678c9,41485,1731042802970 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,425 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure d1ba675678c9%2C46753%2C1731042802851.1731042804603, worker=d1ba675678c9,39291,1731042803020 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,425 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d1ba675678c9%2C46753%2C1731042802851.1731042804603, worker=d1ba675678c9,39291,1731042803020 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,425 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e61dc9e059921fddc5ad7f6f7971acc6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:30,425 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e61dc9e059921fddc5ad7f6f7971acc6, ASSIGN 2024-11-08T05:13:30,425 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=81806cb184c70c9dc0df408694092d85, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:30,425 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=81806cb184c70c9dc0df408694092d85, ASSIGN 2024-11-08T05:13:30,425 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=868527ec322eba1d7cbd6371701f7ad2, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:30,425 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=868527ec322eba1d7cbd6371701f7ad2, ASSIGN 2024-11-08T05:13:30,425 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a12ea80f78b339cfbfc0e18b5338df0c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:30,425 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a12ea80f78b339cfbfc0e18b5338df0c, ASSIGN 2024-11-08T05:13:30,425 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-11-08T05:13:30,426 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=61 2024-11-08T05:13:30,426 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=d1ba675678c9,41485,1731042802970, table=hbase:meta, region=1588230740 2024-11-08T05:13:30,427 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d1ba675678c9,41485,1731042802970, state=OPEN 2024-11-08T05:13:30,439 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39291-0x10118dcefab0003, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T05:13:30,439 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41485-0x10118dcefab0002, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T05:13:30,439 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37587-0x10118dcefab0000, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T05:13:30,439 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T05:13:30,439 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T05:13:30,439 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T05:13:30,440 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e61dc9e059921fddc5ad7f6f7971acc6, ASSIGN to state=OFFLINE, location=null, table=test, region=e61dc9e059921fddc5ad7f6f7971acc6 to restore RIT 2024-11-08T05:13:30,440 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=81806cb184c70c9dc0df408694092d85, ASSIGN to state=OFFLINE, location=null, table=test, region=81806cb184c70c9dc0df408694092d85 to restore RIT 2024-11-08T05:13:30,440 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=868527ec322eba1d7cbd6371701f7ad2, ASSIGN to state=OFFLINE, location=null, table=test, region=868527ec322eba1d7cbd6371701f7ad2 to restore RIT 2024-11-08T05:13:30,440 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a12ea80f78b339cfbfc0e18b5338df0c, ASSIGN to state=OFFLINE, location=null, table=test, region=a12ea80f78b339cfbfc0e18b5338df0c to restore RIT 2024-11-08T05:13:30,442 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731042840442 2024-11-08T05:13:30,442 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-08T05:13:30,442 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure e61dc9e059921fddc5ad7f6f7971acc6, server=d1ba675678c9,41485,1731042802970:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d1ba675678c9,46753,1731042802851, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T05:13:30,444 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-08T05:13:30,445 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-08T05:13:30,445 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=868527ec322eba1d7cbd6371701f7ad2, ASSIGN 2024-11-08T05:13:30,445 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T05:13:30,445 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=868527ec322eba1d7cbd6371701f7ad2, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T05:13:30,449 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6240e38715889f3bb77bb89478c81a6c, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,449 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eeaad0e7e7aa66f0b51c273c0aaf953c, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,449 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=68bbd0353f97ee12228f7798b0b987ea, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,449 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=64ddf686fed30cdb23b2e874efec598d, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,449 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d7680a9929f15640ca61551f186f33c1, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,450 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=65d5312dbd2f96bf3b16bdfd7504dff6, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,450 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9b3274d80dcb7deed617c19ac1de4a68, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,450 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2d23e3e11ecc480e314e989d1bb8d0c3, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,450 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fe1545b16a81758dcb9430dbf98c4ab9, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,450 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a12ea80f78b339cfbfc0e18b5338df0c, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,450 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e1c63297d3a8aea2fdbd1842301c811, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,450 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=40440b5f39bb5421276aa0b156481a9e, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,450 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=27c9aaf34aaf98cd5bc8030630ae7fe3, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,451 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=868527ec322eba1d7cbd6371701f7ad2, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,451 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d951febea78018bb9f10f07d8788575d, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,451 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b0ebd9a9794acc56e430b36d1f17621c, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,451 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7830dc6b1acea6e1d468f8008d8a82b9, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,451 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12f1148a7b8db3e4f515337b8b7746fa, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,451 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e26dcb2339b364f3699d110c15cd4bb3, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,451 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=445b14856951586918c91d4ba342f1ff, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,452 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=14efcc1b9822cc74f4d4a96648555167, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,452 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7c8f72849b2e1719ed903afc73e0571a, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,452 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=81806cb184c70c9dc0df408694092d85, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,452 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=87b3455cffd1f4e06e5901b3bcbed9dc, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,452 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=de358f808110a48fa32e0533fefdc95c, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,452 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e61dc9e059921fddc5ad7f6f7971acc6, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,452 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-08T05:13:30,452 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731042870452 2024-11-08T05:13:30,452 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731042930452 2024-11-08T05:13:30,453 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 10 msec 2024-11-08T05:13:30,453 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-08T05:13:30,453 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T05:13:30,453 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-08T05:13:30,453 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-08T05:13:30,453 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-08T05:13:30,453 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-08T05:13:30,453 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-08T05:13:30,453 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-08T05:13:30,454 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-08T05:13:30,457 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-08T05:13:30,457 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T05:13:30,457 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T05:13:30,457 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T05:13:30,457 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T05:13:30,457 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T05:13:30,457 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T05:13:30,457 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T05:13:30,457 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T05:13:30,457 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T05:13:30,457 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-08T05:13:30,458 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-08T05:13:30,459 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure e61dc9e059921fddc5ad7f6f7971acc6, server=d1ba675678c9,41485,1731042802970:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d1ba675678c9,46753,1731042802851, splitWal=true, meta=true 2024-11-08T05:13:30,459 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure d1ba675678c9%2C46753%2C1731042802851.meta.1731042805109.meta, worker=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:30,459 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d1ba675678c9%2C46753%2C1731042802851.meta.1731042805109.meta, worker=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:30,460 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T05:13:30,460 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d1ba675678c9,41485,1731042802970 2024-11-08T05:13:30,460 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure d1ba675678c9%2C46753%2C1731042802851.1731042804603, worker=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:30,460 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d1ba675678c9%2C46753%2C1731042802851.1731042804603, worker=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:30,460 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e61dc9e059921fddc5ad7f6f7971acc6, ASSIGN 2024-11-08T05:13:30,460 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=81806cb184c70c9dc0df408694092d85, ASSIGN 2024-11-08T05:13:30,460 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=868527ec322eba1d7cbd6371701f7ad2, ASSIGN 2024-11-08T05:13:30,460 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure e61dc9e059921fddc5ad7f6f7971acc6, server=d1ba675678c9,41485,1731042802970:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d1ba675678c9,46753,1731042802851, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:30,460 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure d1ba675678c9%2C46753%2C1731042802851.meta.1731042805109.meta, worker=d1ba675678c9,39291,1731042803020 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,460 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d1ba675678c9%2C46753%2C1731042802851.meta.1731042805109.meta, worker=d1ba675678c9,39291,1731042803020 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,460 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,460 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d1ba675678c9,41485,1731042802970 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,460 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure d1ba675678c9%2C46753%2C1731042802851.1731042804603, worker=d1ba675678c9,39291,1731042803020 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,460 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d1ba675678c9%2C46753%2C1731042802851.1731042804603, worker=d1ba675678c9,39291,1731042803020 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,460 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e61dc9e059921fddc5ad7f6f7971acc6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:30,460 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e61dc9e059921fddc5ad7f6f7971acc6, ASSIGN 2024-11-08T05:13:30,460 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=81806cb184c70c9dc0df408694092d85, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:30,460 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=81806cb184c70c9dc0df408694092d85, ASSIGN 2024-11-08T05:13:30,461 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=868527ec322eba1d7cbd6371701f7ad2, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:30,461 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=868527ec322eba1d7cbd6371701f7ad2, ASSIGN 2024-11-08T05:13:30,461 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-08T05:13:30,462 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=61 2024-11-08T05:13:30,462 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=d1ba675678c9,41485,1731042802970, table=hbase:meta, region=1588230740 2024-11-08T05:13:30,462 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d1ba675678c9,41485,1731042802970, state=OPEN 2024-11-08T05:13:30,471 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37587-0x10118dcefab0000, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T05:13:30,471 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39291-0x10118dcefab0003, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T05:13:30,471 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41485-0x10118dcefab0002, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T05:13:30,471 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T05:13:30,471 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T05:13:30,471 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T05:13:30,472 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e61dc9e059921fddc5ad7f6f7971acc6, ASSIGN to state=OFFLINE, location=null, table=test, region=e61dc9e059921fddc5ad7f6f7971acc6 to restore RIT 2024-11-08T05:13:30,472 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=81806cb184c70c9dc0df408694092d85, ASSIGN to state=OFFLINE, location=null, table=test, region=81806cb184c70c9dc0df408694092d85 to restore RIT 2024-11-08T05:13:30,472 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=868527ec322eba1d7cbd6371701f7ad2, ASSIGN to state=OFFLINE, location=null, table=test, region=868527ec322eba1d7cbd6371701f7ad2 to restore RIT 2024-11-08T05:13:30,472 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731042840472 2024-11-08T05:13:30,472 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-08T05:13:30,472 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure e61dc9e059921fddc5ad7f6f7971acc6, server=d1ba675678c9,41485,1731042802970:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d1ba675678c9,46753,1731042802851, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T05:13:30,474 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-08T05:13:30,476 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-08T05:13:30,476 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=81806cb184c70c9dc0df408694092d85, ASSIGN 2024-11-08T05:13:30,476 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T05:13:30,476 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=81806cb184c70c9dc0df408694092d85, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T05:13:30,478 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6240e38715889f3bb77bb89478c81a6c, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,479 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eeaad0e7e7aa66f0b51c273c0aaf953c, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,479 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=68bbd0353f97ee12228f7798b0b987ea, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,479 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=64ddf686fed30cdb23b2e874efec598d, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,479 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d7680a9929f15640ca61551f186f33c1, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,479 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=65d5312dbd2f96bf3b16bdfd7504dff6, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,479 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9b3274d80dcb7deed617c19ac1de4a68, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,480 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2d23e3e11ecc480e314e989d1bb8d0c3, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,480 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fe1545b16a81758dcb9430dbf98c4ab9, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,480 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a12ea80f78b339cfbfc0e18b5338df0c, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,480 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e1c63297d3a8aea2fdbd1842301c811, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,480 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=40440b5f39bb5421276aa0b156481a9e, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,480 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=27c9aaf34aaf98cd5bc8030630ae7fe3, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,480 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=868527ec322eba1d7cbd6371701f7ad2, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,481 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d951febea78018bb9f10f07d8788575d, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,481 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b0ebd9a9794acc56e430b36d1f17621c, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,481 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7830dc6b1acea6e1d468f8008d8a82b9, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,481 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12f1148a7b8db3e4f515337b8b7746fa, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,481 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e26dcb2339b364f3699d110c15cd4bb3, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,481 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=445b14856951586918c91d4ba342f1ff, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,481 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=14efcc1b9822cc74f4d4a96648555167, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,481 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7c8f72849b2e1719ed903afc73e0571a, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,482 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=81806cb184c70c9dc0df408694092d85, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,482 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=87b3455cffd1f4e06e5901b3bcbed9dc, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,482 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=de358f808110a48fa32e0533fefdc95c, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,482 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e61dc9e059921fddc5ad7f6f7971acc6, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,482 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-08T05:13:30,482 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731042870482 2024-11-08T05:13:30,482 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731042930482 2024-11-08T05:13:30,482 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 9 msec 2024-11-08T05:13:30,482 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-08T05:13:30,482 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T05:13:30,482 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-08T05:13:30,483 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-08T05:13:30,483 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-08T05:13:30,483 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-08T05:13:30,483 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-08T05:13:30,483 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-08T05:13:30,484 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-08T05:13:30,486 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-08T05:13:30,486 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T05:13:30,486 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T05:13:30,486 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T05:13:30,487 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T05:13:30,487 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T05:13:30,487 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T05:13:30,487 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T05:13:30,487 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T05:13:30,487 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-08T05:13:30,488 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-08T05:13:30,489 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure e61dc9e059921fddc5ad7f6f7971acc6, server=d1ba675678c9,41485,1731042802970:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d1ba675678c9,46753,1731042802851, splitWal=true, meta=true 2024-11-08T05:13:30,489 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure d1ba675678c9%2C46753%2C1731042802851.meta.1731042805109.meta, worker=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:30,489 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d1ba675678c9%2C46753%2C1731042802851.meta.1731042805109.meta, worker=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:30,489 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T05:13:30,489 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d1ba675678c9,41485,1731042802970 2024-11-08T05:13:30,489 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure d1ba675678c9%2C46753%2C1731042802851.1731042804603, worker=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:30,489 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d1ba675678c9%2C46753%2C1731042802851.1731042804603, worker=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:30,489 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e61dc9e059921fddc5ad7f6f7971acc6, ASSIGN 2024-11-08T05:13:30,489 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=81806cb184c70c9dc0df408694092d85, ASSIGN 2024-11-08T05:13:30,489 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure e61dc9e059921fddc5ad7f6f7971acc6, server=d1ba675678c9,41485,1731042802970:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d1ba675678c9,46753,1731042802851, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:30,489 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure d1ba675678c9%2C46753%2C1731042802851.meta.1731042805109.meta, worker=d1ba675678c9,39291,1731042803020 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,489 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d1ba675678c9%2C46753%2C1731042802851.meta.1731042805109.meta, worker=d1ba675678c9,39291,1731042803020 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,489 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,489 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d1ba675678c9,41485,1731042802970 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,489 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure d1ba675678c9%2C46753%2C1731042802851.1731042804603, worker=d1ba675678c9,39291,1731042803020 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,490 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d1ba675678c9%2C46753%2C1731042802851.1731042804603, worker=d1ba675678c9,39291,1731042803020 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,490 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e61dc9e059921fddc5ad7f6f7971acc6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:30,490 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e61dc9e059921fddc5ad7f6f7971acc6, ASSIGN 2024-11-08T05:13:30,490 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=81806cb184c70c9dc0df408694092d85, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:30,490 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=81806cb184c70c9dc0df408694092d85, ASSIGN 2024-11-08T05:13:30,490 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-08T05:13:30,491 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=61 2024-11-08T05:13:30,491 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=d1ba675678c9,41485,1731042802970, table=hbase:meta, region=1588230740 2024-11-08T05:13:30,491 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d1ba675678c9,41485,1731042802970, state=OPEN 2024-11-08T05:13:30,495 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37587 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T05:13:30,496 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37587 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T05:13:30,533 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39291-0x10118dcefab0003, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T05:13:30,533 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41485-0x10118dcefab0002, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T05:13:30,533 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37587-0x10118dcefab0000, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T05:13:30,533 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T05:13:30,533 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T05:13:30,533 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T05:13:30,534 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e61dc9e059921fddc5ad7f6f7971acc6, ASSIGN to state=OFFLINE, location=null, table=test, region=e61dc9e059921fddc5ad7f6f7971acc6 to restore RIT 2024-11-08T05:13:30,534 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=81806cb184c70c9dc0df408694092d85, ASSIGN to state=OFFLINE, location=null, table=test, region=81806cb184c70c9dc0df408694092d85 to restore RIT 2024-11-08T05:13:30,535 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731042840535 2024-11-08T05:13:30,535 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-08T05:13:30,535 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure e61dc9e059921fddc5ad7f6f7971acc6, server=d1ba675678c9,41485,1731042802970:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d1ba675678c9,46753,1731042802851, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T05:13:30,536 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-08T05:13:30,537 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-08T05:13:30,537 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e61dc9e059921fddc5ad7f6f7971acc6, ASSIGN 2024-11-08T05:13:30,537 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T05:13:30,537 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e61dc9e059921fddc5ad7f6f7971acc6, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T05:13:30,540 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6240e38715889f3bb77bb89478c81a6c, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,540 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eeaad0e7e7aa66f0b51c273c0aaf953c, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,540 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=68bbd0353f97ee12228f7798b0b987ea, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,540 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=64ddf686fed30cdb23b2e874efec598d, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,540 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d7680a9929f15640ca61551f186f33c1, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,540 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=65d5312dbd2f96bf3b16bdfd7504dff6, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,541 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9b3274d80dcb7deed617c19ac1de4a68, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,541 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2d23e3e11ecc480e314e989d1bb8d0c3, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,541 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fe1545b16a81758dcb9430dbf98c4ab9, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,541 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a12ea80f78b339cfbfc0e18b5338df0c, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,541 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e1c63297d3a8aea2fdbd1842301c811, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,541 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=40440b5f39bb5421276aa0b156481a9e, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,541 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=27c9aaf34aaf98cd5bc8030630ae7fe3, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,541 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=868527ec322eba1d7cbd6371701f7ad2, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,541 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d951febea78018bb9f10f07d8788575d, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,542 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b0ebd9a9794acc56e430b36d1f17621c, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,542 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7830dc6b1acea6e1d468f8008d8a82b9, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,542 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12f1148a7b8db3e4f515337b8b7746fa, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,542 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e26dcb2339b364f3699d110c15cd4bb3, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,542 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=445b14856951586918c91d4ba342f1ff, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,542 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=14efcc1b9822cc74f4d4a96648555167, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,542 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7c8f72849b2e1719ed903afc73e0571a, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,542 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=81806cb184c70c9dc0df408694092d85, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,543 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=87b3455cffd1f4e06e5901b3bcbed9dc, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,543 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=de358f808110a48fa32e0533fefdc95c, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,543 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e61dc9e059921fddc5ad7f6f7971acc6, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,543 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-08T05:13:30,543 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731042870543 2024-11-08T05:13:30,543 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731042930543 2024-11-08T05:13:30,543 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 8 msec 2024-11-08T05:13:30,543 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-08T05:13:30,543 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T05:13:30,543 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-08T05:13:30,543 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-08T05:13:30,544 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-08T05:13:30,544 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-08T05:13:30,544 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-08T05:13:30,544 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-08T05:13:30,545 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-08T05:13:30,547 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-08T05:13:30,547 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T05:13:30,547 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T05:13:30,547 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T05:13:30,547 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T05:13:30,547 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T05:13:30,547 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T05:13:30,547 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T05:13:30,548 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-08T05:13:30,548 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-08T05:13:30,550 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure e61dc9e059921fddc5ad7f6f7971acc6, server=d1ba675678c9,41485,1731042802970:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d1ba675678c9,46753,1731042802851, splitWal=true, meta=true 2024-11-08T05:13:30,550 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure d1ba675678c9%2C46753%2C1731042802851.meta.1731042805109.meta, worker=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:30,550 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d1ba675678c9%2C46753%2C1731042802851.meta.1731042805109.meta, worker=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:30,550 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T05:13:30,550 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d1ba675678c9,41485,1731042802970 2024-11-08T05:13:30,550 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure d1ba675678c9%2C46753%2C1731042802851.1731042804603, worker=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:30,550 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d1ba675678c9%2C46753%2C1731042802851.1731042804603, worker=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:30,550 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e61dc9e059921fddc5ad7f6f7971acc6, ASSIGN 2024-11-08T05:13:30,550 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure e61dc9e059921fddc5ad7f6f7971acc6, server=d1ba675678c9,41485,1731042802970:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d1ba675678c9,46753,1731042802851, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:30,550 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure d1ba675678c9%2C46753%2C1731042802851.meta.1731042805109.meta, worker=d1ba675678c9,39291,1731042803020 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,550 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d1ba675678c9%2C46753%2C1731042802851.meta.1731042805109.meta, worker=d1ba675678c9,39291,1731042803020 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,550 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,550 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d1ba675678c9,41485,1731042802970 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,550 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure d1ba675678c9%2C46753%2C1731042802851.1731042804603, worker=d1ba675678c9,39291,1731042803020 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,550 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d1ba675678c9%2C46753%2C1731042802851.1731042804603, worker=d1ba675678c9,39291,1731042803020 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,550 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e61dc9e059921fddc5ad7f6f7971acc6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:30,550 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e61dc9e059921fddc5ad7f6f7971acc6, ASSIGN 2024-11-08T05:13:30,551 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-08T05:13:30,551 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=61 2024-11-08T05:13:30,552 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=d1ba675678c9,41485,1731042802970, table=hbase:meta, region=1588230740 2024-11-08T05:13:30,552 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d1ba675678c9,41485,1731042802970, state=OPEN 2024-11-08T05:13:30,573 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39291-0x10118dcefab0003, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T05:13:30,573 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37587-0x10118dcefab0000, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T05:13:30,573 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41485-0x10118dcefab0002, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T05:13:30,573 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T05:13:30,573 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T05:13:30,573 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T05:13:30,573 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e61dc9e059921fddc5ad7f6f7971acc6, ASSIGN to state=OFFLINE, location=null, table=test, region=e61dc9e059921fddc5ad7f6f7971acc6 to restore RIT 2024-11-08T05:13:30,574 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731042840574 2024-11-08T05:13:30,574 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-08T05:13:30,574 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure e61dc9e059921fddc5ad7f6f7971acc6, server=d1ba675678c9,41485,1731042802970:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d1ba675678c9,46753,1731042802851, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T05:13:30,576 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-08T05:13:30,577 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-08T05:13:30,577 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d1ba675678c9%2C46753%2C1731042802851.1731042804603, worker=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:30,577 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T05:13:30,577 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d1ba675678c9%2C46753%2C1731042802851.1731042804603, worker=d1ba675678c9,39291,1731042803020 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T05:13:30,578 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6240e38715889f3bb77bb89478c81a6c, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,579 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eeaad0e7e7aa66f0b51c273c0aaf953c, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,579 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=68bbd0353f97ee12228f7798b0b987ea, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,579 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=64ddf686fed30cdb23b2e874efec598d, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,579 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d7680a9929f15640ca61551f186f33c1, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,579 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=65d5312dbd2f96bf3b16bdfd7504dff6, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,579 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9b3274d80dcb7deed617c19ac1de4a68, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,579 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2d23e3e11ecc480e314e989d1bb8d0c3, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,579 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fe1545b16a81758dcb9430dbf98c4ab9, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,579 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a12ea80f78b339cfbfc0e18b5338df0c, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,579 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e1c63297d3a8aea2fdbd1842301c811, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,579 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=40440b5f39bb5421276aa0b156481a9e, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,579 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=27c9aaf34aaf98cd5bc8030630ae7fe3, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,580 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=868527ec322eba1d7cbd6371701f7ad2, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,580 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d951febea78018bb9f10f07d8788575d, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,580 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b0ebd9a9794acc56e430b36d1f17621c, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,580 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7830dc6b1acea6e1d468f8008d8a82b9, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,580 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12f1148a7b8db3e4f515337b8b7746fa, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,580 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e26dcb2339b364f3699d110c15cd4bb3, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,580 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=445b14856951586918c91d4ba342f1ff, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,580 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=14efcc1b9822cc74f4d4a96648555167, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,580 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7c8f72849b2e1719ed903afc73e0571a, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,580 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=81806cb184c70c9dc0df408694092d85, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,580 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=87b3455cffd1f4e06e5901b3bcbed9dc, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,580 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=de358f808110a48fa32e0533fefdc95c, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,581 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e61dc9e059921fddc5ad7f6f7971acc6, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,581 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-08T05:13:30,581 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731042870581 2024-11-08T05:13:30,581 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731042930581 2024-11-08T05:13:30,581 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 6 msec 2024-11-08T05:13:30,581 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-08T05:13:30,581 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T05:13:30,581 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-08T05:13:30,581 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-08T05:13:30,581 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-08T05:13:30,581 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-08T05:13:30,581 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-08T05:13:30,581 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-08T05:13:30,582 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-08T05:13:30,584 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-08T05:13:30,584 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T05:13:30,584 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T05:13:30,584 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T05:13:30,584 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T05:13:30,584 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T05:13:30,584 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T05:13:30,584 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-08T05:13:30,585 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-08T05:13:30,585 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure e61dc9e059921fddc5ad7f6f7971acc6, server=d1ba675678c9,41485,1731042802970:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d1ba675678c9,46753,1731042802851, splitWal=true, meta=true 2024-11-08T05:13:30,585 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure d1ba675678c9%2C46753%2C1731042802851.meta.1731042805109.meta, worker=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:30,585 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d1ba675678c9%2C46753%2C1731042802851.meta.1731042805109.meta, worker=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:30,585 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T05:13:30,586 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d1ba675678c9,41485,1731042802970 2024-11-08T05:13:30,586 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure d1ba675678c9%2C46753%2C1731042802851.1731042804603, worker=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:30,586 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d1ba675678c9%2C46753%2C1731042802851.1731042804603, worker=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:30,586 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure e61dc9e059921fddc5ad7f6f7971acc6, server=d1ba675678c9,41485,1731042802970:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d1ba675678c9,46753,1731042802851, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:30,586 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure d1ba675678c9%2C46753%2C1731042802851.meta.1731042805109.meta, worker=d1ba675678c9,39291,1731042803020 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,586 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d1ba675678c9%2C46753%2C1731042802851.meta.1731042805109.meta, worker=d1ba675678c9,39291,1731042803020 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,586 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,586 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d1ba675678c9,41485,1731042802970 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,586 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure d1ba675678c9%2C46753%2C1731042802851.1731042804603, worker=d1ba675678c9,39291,1731042803020 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,586 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d1ba675678c9%2C46753%2C1731042802851.1731042804603, worker=d1ba675678c9,39291,1731042803020 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,586 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-08T05:13:30,587 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=61 2024-11-08T05:13:30,587 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=d1ba675678c9,41485,1731042802970, table=hbase:meta, region=1588230740 2024-11-08T05:13:30,587 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d1ba675678c9,41485,1731042802970, state=OPEN 2024-11-08T05:13:30,597 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37587-0x10118dcefab0000, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T05:13:30,597 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41485-0x10118dcefab0002, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T05:13:30,597 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39291-0x10118dcefab0003, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T05:13:30,597 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T05:13:30,597 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T05:13:30,597 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T05:13:30,598 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731042840598 2024-11-08T05:13:30,598 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-08T05:13:30,598 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure e61dc9e059921fddc5ad7f6f7971acc6, server=d1ba675678c9,41485,1731042802970:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d1ba675678c9,46753,1731042802851, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T05:13:30,599 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-08T05:13:30,600 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-08T05:13:30,601 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure d1ba675678c9%2C46753%2C1731042802851.1731042804603, worker=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:30,601 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T05:13:30,601 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure d1ba675678c9%2C46753%2C1731042802851.1731042804603, worker=d1ba675678c9,39291,1731042803020 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T05:13:30,603 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6240e38715889f3bb77bb89478c81a6c, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,604 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eeaad0e7e7aa66f0b51c273c0aaf953c, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,604 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=68bbd0353f97ee12228f7798b0b987ea, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,604 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=64ddf686fed30cdb23b2e874efec598d, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,604 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d7680a9929f15640ca61551f186f33c1, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,604 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=65d5312dbd2f96bf3b16bdfd7504dff6, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,604 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9b3274d80dcb7deed617c19ac1de4a68, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,604 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2d23e3e11ecc480e314e989d1bb8d0c3, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,604 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fe1545b16a81758dcb9430dbf98c4ab9, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,604 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a12ea80f78b339cfbfc0e18b5338df0c, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,604 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e1c63297d3a8aea2fdbd1842301c811, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,604 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=40440b5f39bb5421276aa0b156481a9e, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,605 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=27c9aaf34aaf98cd5bc8030630ae7fe3, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,605 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=868527ec322eba1d7cbd6371701f7ad2, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,605 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d951febea78018bb9f10f07d8788575d, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,605 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b0ebd9a9794acc56e430b36d1f17621c, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,605 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7830dc6b1acea6e1d468f8008d8a82b9, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,605 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12f1148a7b8db3e4f515337b8b7746fa, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,605 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e26dcb2339b364f3699d110c15cd4bb3, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,605 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=445b14856951586918c91d4ba342f1ff, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,605 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=14efcc1b9822cc74f4d4a96648555167, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,605 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7c8f72849b2e1719ed903afc73e0571a, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,605 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=81806cb184c70c9dc0df408694092d85, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,605 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=87b3455cffd1f4e06e5901b3bcbed9dc, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,606 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=de358f808110a48fa32e0533fefdc95c, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,606 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e61dc9e059921fddc5ad7f6f7971acc6, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,606 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-08T05:13:30,606 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731042870606 2024-11-08T05:13:30,606 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731042930606 2024-11-08T05:13:30,606 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 7 msec 2024-11-08T05:13:30,606 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-08T05:13:30,606 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T05:13:30,606 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-08T05:13:30,606 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-08T05:13:30,606 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-08T05:13:30,606 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-08T05:13:30,606 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-08T05:13:30,607 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-08T05:13:30,607 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-08T05:13:30,609 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-08T05:13:30,609 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T05:13:30,609 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T05:13:30,609 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T05:13:30,609 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T05:13:30,609 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T05:13:30,609 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-08T05:13:30,610 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-08T05:13:30,610 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure e61dc9e059921fddc5ad7f6f7971acc6, server=d1ba675678c9,41485,1731042802970:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d1ba675678c9,46753,1731042802851, splitWal=true, meta=true 2024-11-08T05:13:30,610 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure d1ba675678c9%2C46753%2C1731042802851.meta.1731042805109.meta, worker=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:30,610 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d1ba675678c9%2C46753%2C1731042802851.meta.1731042805109.meta, worker=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:30,610 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T05:13:30,610 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d1ba675678c9,41485,1731042802970 2024-11-08T05:13:30,610 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure d1ba675678c9%2C46753%2C1731042802851.1731042804603, worker=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:30,610 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure e61dc9e059921fddc5ad7f6f7971acc6, server=d1ba675678c9,41485,1731042802970:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d1ba675678c9,46753,1731042802851, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:30,611 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure d1ba675678c9%2C46753%2C1731042802851.meta.1731042805109.meta, worker=d1ba675678c9,39291,1731042803020 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,611 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d1ba675678c9%2C46753%2C1731042802851.meta.1731042805109.meta, worker=d1ba675678c9,39291,1731042803020 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,611 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,611 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d1ba675678c9,41485,1731042802970 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,611 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure d1ba675678c9%2C46753%2C1731042802851.1731042804603, worker=d1ba675678c9,39291,1731042803020 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,611 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-08T05:13:30,611 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=61 2024-11-08T05:13:30,612 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=d1ba675678c9,41485,1731042802970, table=hbase:meta, region=1588230740 2024-11-08T05:13:30,612 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d1ba675678c9,41485,1731042802970, state=OPEN 2024-11-08T05:13:30,618 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41485-0x10118dcefab0002, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T05:13:30,618 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37587-0x10118dcefab0000, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T05:13:30,618 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39291-0x10118dcefab0003, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T05:13:30,618 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T05:13:30,618 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T05:13:30,618 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T05:13:30,619 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731042840619 2024-11-08T05:13:30,619 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-08T05:13:30,619 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure e61dc9e059921fddc5ad7f6f7971acc6, server=d1ba675678c9,41485,1731042802970:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d1ba675678c9,46753,1731042802851, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T05:13:30,621 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-08T05:13:30,622 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-08T05:13:30,622 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d1ba675678c9,41485,1731042802970 2024-11-08T05:13:30,622 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T05:13:30,623 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d1ba675678c9,41485,1731042802970 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T05:13:30,625 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6240e38715889f3bb77bb89478c81a6c, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,625 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eeaad0e7e7aa66f0b51c273c0aaf953c, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,625 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=68bbd0353f97ee12228f7798b0b987ea, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,625 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=64ddf686fed30cdb23b2e874efec598d, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,625 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d7680a9929f15640ca61551f186f33c1, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,625 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=65d5312dbd2f96bf3b16bdfd7504dff6, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,626 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9b3274d80dcb7deed617c19ac1de4a68, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,626 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2d23e3e11ecc480e314e989d1bb8d0c3, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,626 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fe1545b16a81758dcb9430dbf98c4ab9, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,626 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a12ea80f78b339cfbfc0e18b5338df0c, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,626 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e1c63297d3a8aea2fdbd1842301c811, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,626 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=40440b5f39bb5421276aa0b156481a9e, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,626 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=27c9aaf34aaf98cd5bc8030630ae7fe3, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,626 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=868527ec322eba1d7cbd6371701f7ad2, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,627 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d951febea78018bb9f10f07d8788575d, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,627 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b0ebd9a9794acc56e430b36d1f17621c, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,627 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7830dc6b1acea6e1d468f8008d8a82b9, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,627 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12f1148a7b8db3e4f515337b8b7746fa, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,627 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e26dcb2339b364f3699d110c15cd4bb3, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,627 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=445b14856951586918c91d4ba342f1ff, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,627 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=14efcc1b9822cc74f4d4a96648555167, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,627 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7c8f72849b2e1719ed903afc73e0571a, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,627 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=81806cb184c70c9dc0df408694092d85, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,628 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=87b3455cffd1f4e06e5901b3bcbed9dc, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,628 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=de358f808110a48fa32e0533fefdc95c, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,628 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e61dc9e059921fddc5ad7f6f7971acc6, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,628 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-08T05:13:30,628 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731042870628 2024-11-08T05:13:30,628 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731042930628 2024-11-08T05:13:30,628 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 8 msec 2024-11-08T05:13:30,628 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-08T05:13:30,628 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T05:13:30,628 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-08T05:13:30,628 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-08T05:13:30,629 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-08T05:13:30,629 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-08T05:13:30,629 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-08T05:13:30,629 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-08T05:13:30,630 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-08T05:13:30,632 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-08T05:13:30,632 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T05:13:30,632 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T05:13:30,632 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T05:13:30,632 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T05:13:30,632 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-08T05:13:30,633 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-08T05:13:30,634 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure e61dc9e059921fddc5ad7f6f7971acc6, server=d1ba675678c9,41485,1731042802970:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d1ba675678c9,46753,1731042802851, splitWal=true, meta=true 2024-11-08T05:13:30,634 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure d1ba675678c9%2C46753%2C1731042802851.meta.1731042805109.meta, worker=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:30,634 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d1ba675678c9%2C46753%2C1731042802851.meta.1731042805109.meta, worker=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:30,634 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T05:13:30,634 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d1ba675678c9,41485,1731042802970 2024-11-08T05:13:30,634 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure e61dc9e059921fddc5ad7f6f7971acc6, server=d1ba675678c9,41485,1731042802970:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d1ba675678c9,46753,1731042802851, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:30,634 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure d1ba675678c9%2C46753%2C1731042802851.meta.1731042805109.meta, worker=d1ba675678c9,39291,1731042803020 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,634 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d1ba675678c9%2C46753%2C1731042802851.meta.1731042805109.meta, worker=d1ba675678c9,39291,1731042803020 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,634 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,634 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d1ba675678c9,41485,1731042802970 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,634 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-08T05:13:30,635 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=61 2024-11-08T05:13:30,635 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=d1ba675678c9,41485,1731042802970, table=hbase:meta, region=1588230740 2024-11-08T05:13:30,635 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37587 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T05:13:30,635 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d1ba675678c9,41485,1731042802970, state=OPEN 2024-11-08T05:13:30,636 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37587 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T05:13:30,646 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37587-0x10118dcefab0000, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T05:13:30,646 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41485-0x10118dcefab0002, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T05:13:30,646 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39291-0x10118dcefab0003, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T05:13:30,646 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T05:13:30,646 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T05:13:30,646 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T05:13:30,647 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731042840647 2024-11-08T05:13:30,647 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-08T05:13:30,647 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure e61dc9e059921fddc5ad7f6f7971acc6, server=d1ba675678c9,41485,1731042802970:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d1ba675678c9,46753,1731042802851, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T05:13:30,649 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-08T05:13:30,650 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-08T05:13:30,650 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T05:13:30,650 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T05:13:30,650 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T05:13:30,652 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6240e38715889f3bb77bb89478c81a6c, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,652 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eeaad0e7e7aa66f0b51c273c0aaf953c, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,652 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=68bbd0353f97ee12228f7798b0b987ea, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,652 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=64ddf686fed30cdb23b2e874efec598d, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,652 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d7680a9929f15640ca61551f186f33c1, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,652 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=65d5312dbd2f96bf3b16bdfd7504dff6, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,652 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9b3274d80dcb7deed617c19ac1de4a68, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,652 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2d23e3e11ecc480e314e989d1bb8d0c3, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,652 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fe1545b16a81758dcb9430dbf98c4ab9, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,653 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a12ea80f78b339cfbfc0e18b5338df0c, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,653 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e1c63297d3a8aea2fdbd1842301c811, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,653 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=40440b5f39bb5421276aa0b156481a9e, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,653 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=27c9aaf34aaf98cd5bc8030630ae7fe3, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,653 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=868527ec322eba1d7cbd6371701f7ad2, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,653 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d951febea78018bb9f10f07d8788575d, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,653 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b0ebd9a9794acc56e430b36d1f17621c, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,653 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7830dc6b1acea6e1d468f8008d8a82b9, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,653 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12f1148a7b8db3e4f515337b8b7746fa, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,653 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e26dcb2339b364f3699d110c15cd4bb3, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,654 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=445b14856951586918c91d4ba342f1ff, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,654 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=14efcc1b9822cc74f4d4a96648555167, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,654 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7c8f72849b2e1719ed903afc73e0571a, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,654 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=81806cb184c70c9dc0df408694092d85, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,654 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=87b3455cffd1f4e06e5901b3bcbed9dc, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,654 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=de358f808110a48fa32e0533fefdc95c, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,654 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e61dc9e059921fddc5ad7f6f7971acc6, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,654 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-08T05:13:30,654 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731042870654 2024-11-08T05:13:30,654 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731042930654 2024-11-08T05:13:30,654 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 6 msec 2024-11-08T05:13:30,654 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-08T05:13:30,654 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T05:13:30,654 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-08T05:13:30,654 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-08T05:13:30,655 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-08T05:13:30,655 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-08T05:13:30,655 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-08T05:13:30,655 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-08T05:13:30,656 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-08T05:13:30,657 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-08T05:13:30,657 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T05:13:30,657 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T05:13:30,657 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T05:13:30,657 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-08T05:13:30,658 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-08T05:13:30,658 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure e61dc9e059921fddc5ad7f6f7971acc6, server=d1ba675678c9,41485,1731042802970:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d1ba675678c9,46753,1731042802851, splitWal=true, meta=true 2024-11-08T05:13:30,658 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure d1ba675678c9%2C46753%2C1731042802851.meta.1731042805109.meta, worker=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:30,658 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d1ba675678c9%2C46753%2C1731042802851.meta.1731042805109.meta, worker=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:30,658 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T05:13:30,658 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure e61dc9e059921fddc5ad7f6f7971acc6, server=d1ba675678c9,41485,1731042802970:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d1ba675678c9,46753,1731042802851, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:30,658 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure d1ba675678c9%2C46753%2C1731042802851.meta.1731042805109.meta, worker=d1ba675678c9,39291,1731042803020 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,658 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d1ba675678c9%2C46753%2C1731042802851.meta.1731042805109.meta, worker=d1ba675678c9,39291,1731042803020 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,658 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,658 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-11-08T05:13:30,660 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T05:13:30,661 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=61 2024-11-08T05:13:30,661 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=d1ba675678c9,41485,1731042802970, table=hbase:meta, region=1588230740 2024-11-08T05:13:30,661 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d1ba675678c9,41485,1731042802970, state=OPEN 2024-11-08T05:13:30,671 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37587-0x10118dcefab0000, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T05:13:30,671 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41485-0x10118dcefab0002, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T05:13:30,671 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39291-0x10118dcefab0003, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T05:13:30,671 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T05:13:30,671 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T05:13:30,671 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T05:13:30,672 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731042840672 2024-11-08T05:13:30,673 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-08T05:13:30,673 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure e61dc9e059921fddc5ad7f6f7971acc6, server=d1ba675678c9,41485,1731042802970:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d1ba675678c9,46753,1731042802851, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T05:13:30,674 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-08T05:13:30,676 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-08T05:13:30,676 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d1ba675678c9%2C46753%2C1731042802851.meta.1731042805109.meta, worker=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:30,676 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T05:13:30,676 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d1ba675678c9%2C46753%2C1731042802851.meta.1731042805109.meta, worker=d1ba675678c9,39291,1731042803020 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T05:13:30,678 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6240e38715889f3bb77bb89478c81a6c, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,678 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eeaad0e7e7aa66f0b51c273c0aaf953c, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,678 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=68bbd0353f97ee12228f7798b0b987ea, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,678 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=64ddf686fed30cdb23b2e874efec598d, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,678 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d7680a9929f15640ca61551f186f33c1, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,678 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=65d5312dbd2f96bf3b16bdfd7504dff6, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,679 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9b3274d80dcb7deed617c19ac1de4a68, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,679 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2d23e3e11ecc480e314e989d1bb8d0c3, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,679 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fe1545b16a81758dcb9430dbf98c4ab9, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,679 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a12ea80f78b339cfbfc0e18b5338df0c, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,679 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e1c63297d3a8aea2fdbd1842301c811, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,679 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=40440b5f39bb5421276aa0b156481a9e, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,679 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=27c9aaf34aaf98cd5bc8030630ae7fe3, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,679 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=868527ec322eba1d7cbd6371701f7ad2, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,679 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d951febea78018bb9f10f07d8788575d, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,679 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b0ebd9a9794acc56e430b36d1f17621c, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,679 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7830dc6b1acea6e1d468f8008d8a82b9, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,679 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12f1148a7b8db3e4f515337b8b7746fa, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,679 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e26dcb2339b364f3699d110c15cd4bb3, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,680 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=445b14856951586918c91d4ba342f1ff, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,680 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=14efcc1b9822cc74f4d4a96648555167, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,680 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7c8f72849b2e1719ed903afc73e0571a, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,680 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=81806cb184c70c9dc0df408694092d85, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,680 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=87b3455cffd1f4e06e5901b3bcbed9dc, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,680 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=de358f808110a48fa32e0533fefdc95c, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,680 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e61dc9e059921fddc5ad7f6f7971acc6, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,680 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-08T05:13:30,680 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731042870680 2024-11-08T05:13:30,680 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731042930680 2024-11-08T05:13:30,680 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 7 msec 2024-11-08T05:13:30,680 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-08T05:13:30,680 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T05:13:30,680 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-08T05:13:30,680 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-08T05:13:30,681 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-08T05:13:30,681 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-08T05:13:30,681 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-08T05:13:30,681 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-08T05:13:30,682 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-08T05:13:30,683 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-08T05:13:30,683 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T05:13:30,683 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T05:13:30,683 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-08T05:13:30,683 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-08T05:13:30,684 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure e61dc9e059921fddc5ad7f6f7971acc6, server=d1ba675678c9,41485,1731042802970:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d1ba675678c9,46753,1731042802851, splitWal=true, meta=true 2024-11-08T05:13:30,684 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure d1ba675678c9%2C46753%2C1731042802851.meta.1731042805109.meta, worker=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:30,684 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d1ba675678c9%2C46753%2C1731042802851.meta.1731042805109.meta, worker=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:30,684 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure e61dc9e059921fddc5ad7f6f7971acc6, server=d1ba675678c9,41485,1731042802970:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d1ba675678c9,46753,1731042802851, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:30,684 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure d1ba675678c9%2C46753%2C1731042802851.meta.1731042805109.meta, worker=d1ba675678c9,39291,1731042803020 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,684 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d1ba675678c9%2C46753%2C1731042802851.meta.1731042805109.meta, worker=d1ba675678c9,39291,1731042803020 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,684 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-11-08T05:13:30,685 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=61 2024-11-08T05:13:30,685 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=d1ba675678c9,41485,1731042802970, table=hbase:meta, region=1588230740 2024-11-08T05:13:30,685 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d1ba675678c9,41485,1731042802970, state=OPEN 2024-11-08T05:13:30,692 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39291-0x10118dcefab0003, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T05:13:30,692 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41485-0x10118dcefab0002, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T05:13:30,692 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37587-0x10118dcefab0000, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T05:13:30,692 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T05:13:30,692 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T05:13:30,692 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T05:13:30,696 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731042840696 2024-11-08T05:13:30,696 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-08T05:13:30,696 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure e61dc9e059921fddc5ad7f6f7971acc6, server=d1ba675678c9,41485,1731042802970:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d1ba675678c9,46753,1731042802851, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T05:13:30,699 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-08T05:13:30,701 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-08T05:13:30,701 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure d1ba675678c9%2C46753%2C1731042802851.meta.1731042805109.meta, worker=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:30,701 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T05:13:30,701 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure d1ba675678c9%2C46753%2C1731042802851.meta.1731042805109.meta, worker=d1ba675678c9,39291,1731042803020 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T05:13:30,707 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6240e38715889f3bb77bb89478c81a6c, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,707 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eeaad0e7e7aa66f0b51c273c0aaf953c, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,707 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=68bbd0353f97ee12228f7798b0b987ea, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,707 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=64ddf686fed30cdb23b2e874efec598d, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,708 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d7680a9929f15640ca61551f186f33c1, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,708 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=65d5312dbd2f96bf3b16bdfd7504dff6, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,708 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9b3274d80dcb7deed617c19ac1de4a68, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,708 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2d23e3e11ecc480e314e989d1bb8d0c3, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,708 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fe1545b16a81758dcb9430dbf98c4ab9, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,708 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a12ea80f78b339cfbfc0e18b5338df0c, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,708 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e1c63297d3a8aea2fdbd1842301c811, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,709 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=40440b5f39bb5421276aa0b156481a9e, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,709 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=27c9aaf34aaf98cd5bc8030630ae7fe3, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,709 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=868527ec322eba1d7cbd6371701f7ad2, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,709 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d951febea78018bb9f10f07d8788575d, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,709 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b0ebd9a9794acc56e430b36d1f17621c, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,709 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7830dc6b1acea6e1d468f8008d8a82b9, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,709 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12f1148a7b8db3e4f515337b8b7746fa, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,709 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e26dcb2339b364f3699d110c15cd4bb3, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,710 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=445b14856951586918c91d4ba342f1ff, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,710 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=14efcc1b9822cc74f4d4a96648555167, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,710 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7c8f72849b2e1719ed903afc73e0571a, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,710 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=81806cb184c70c9dc0df408694092d85, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,710 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=87b3455cffd1f4e06e5901b3bcbed9dc, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,710 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=de358f808110a48fa32e0533fefdc95c, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,710 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e61dc9e059921fddc5ad7f6f7971acc6, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,710 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-08T05:13:30,711 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731042870710 2024-11-08T05:13:30,711 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731042930711 2024-11-08T05:13:30,711 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 14 msec 2024-11-08T05:13:30,711 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-08T05:13:30,711 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T05:13:30,711 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-08T05:13:30,711 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-08T05:13:30,711 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-08T05:13:30,711 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-08T05:13:30,711 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-08T05:13:30,712 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-08T05:13:30,713 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-08T05:13:30,714 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-08T05:13:30,714 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T05:13:30,714 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-08T05:13:30,715 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-08T05:13:30,715 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure e61dc9e059921fddc5ad7f6f7971acc6, server=d1ba675678c9,41485,1731042802970:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d1ba675678c9,46753,1731042802851, splitWal=true, meta=true 2024-11-08T05:13:30,715 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure d1ba675678c9%2C46753%2C1731042802851.meta.1731042805109.meta, worker=d1ba675678c9,39291,1731042803020 2024-11-08T05:13:30,715 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure e61dc9e059921fddc5ad7f6f7971acc6, server=d1ba675678c9,41485,1731042802970:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d1ba675678c9,46753,1731042802851, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:30,715 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure d1ba675678c9%2C46753%2C1731042802851.meta.1731042805109.meta, worker=d1ba675678c9,39291,1731042803020 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T05:13:30,715 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-11-08T05:13:30,716 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=61 2024-11-08T05:13:30,716 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=d1ba675678c9,41485,1731042802970, table=hbase:meta, region=1588230740 2024-11-08T05:13:30,716 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d1ba675678c9,41485,1731042802970, state=OPEN 2024-11-08T05:13:30,723 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41485-0x10118dcefab0002, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T05:13:30,723 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39291-0x10118dcefab0003, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T05:13:30,723 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37587-0x10118dcefab0000, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T05:13:30,724 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T05:13:30,724 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T05:13:30,724 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T05:13:30,725 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731042840725 2024-11-08T05:13:30,725 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-08T05:13:30,725 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure e61dc9e059921fddc5ad7f6f7971acc6, server=d1ba675678c9,41485,1731042802970:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d1ba675678c9,46753,1731042802851, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T05:13:30,727 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-08T05:13:30,728 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-08T05:13:30,728 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure e61dc9e059921fddc5ad7f6f7971acc6, server=d1ba675678c9,41485,1731042802970:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d1ba675678c9,46753,1731042802851, splitWal=true, meta=true 2024-11-08T05:13:30,728 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T05:13:30,728 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure e61dc9e059921fddc5ad7f6f7971acc6, server=d1ba675678c9,41485,1731042802970:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d1ba675678c9,46753,1731042802851, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1604) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T05:13:30,729 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6240e38715889f3bb77bb89478c81a6c, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,729 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eeaad0e7e7aa66f0b51c273c0aaf953c, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,729 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=68bbd0353f97ee12228f7798b0b987ea, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,729 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'test' 2024-11-08T05:13:30,729 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=64ddf686fed30cdb23b2e874efec598d, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,729 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d7680a9929f15640ca61551f186f33c1, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,730 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=65d5312dbd2f96bf3b16bdfd7504dff6, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,730 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9b3274d80dcb7deed617c19ac1de4a68, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,730 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2d23e3e11ecc480e314e989d1bb8d0c3, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,730 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fe1545b16a81758dcb9430dbf98c4ab9, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,730 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a12ea80f78b339cfbfc0e18b5338df0c, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,730 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e1c63297d3a8aea2fdbd1842301c811, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,730 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=40440b5f39bb5421276aa0b156481a9e, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,730 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=27c9aaf34aaf98cd5bc8030630ae7fe3, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,730 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=868527ec322eba1d7cbd6371701f7ad2, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,730 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d951febea78018bb9f10f07d8788575d, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,730 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b0ebd9a9794acc56e430b36d1f17621c, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,730 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7830dc6b1acea6e1d468f8008d8a82b9, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,731 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12f1148a7b8db3e4f515337b8b7746fa, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,731 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e26dcb2339b364f3699d110c15cd4bb3, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,731 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=445b14856951586918c91d4ba342f1ff, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,731 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=14efcc1b9822cc74f4d4a96648555167, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,731 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7c8f72849b2e1719ed903afc73e0571a, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,731 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=81806cb184c70c9dc0df408694092d85, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,731 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=87b3455cffd1f4e06e5901b3bcbed9dc, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,731 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=de358f808110a48fa32e0533fefdc95c, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,731 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e61dc9e059921fddc5ad7f6f7971acc6, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,731 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-08T05:13:30,731 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731042870731 2024-11-08T05:13:30,731 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731042930731 2024-11-08T05:13:30,731 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 6 msec 2024-11-08T05:13:30,731 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-08T05:13:30,732 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T05:13:30,732 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-08T05:13:30,732 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-08T05:13:30,732 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-08T05:13:30,732 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-08T05:13:30,732 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-08T05:13:30,732 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-08T05:13:30,733 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-08T05:13:30,734 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-08T05:13:30,734 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-08T05:13:30,735 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-08T05:13:30,735 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure e61dc9e059921fddc5ad7f6f7971acc6, server=d1ba675678c9,41485,1731042802970:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d1ba675678c9,46753,1731042802851, splitWal=true, meta=true 2024-11-08T05:13:30,735 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure e61dc9e059921fddc5ad7f6f7971acc6, server=d1ba675678c9,41485,1731042802970:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d1ba675678c9,46753,1731042802851, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-08T05:13:30,735 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-11-08T05:13:30,736 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=61 2024-11-08T05:13:30,736 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=d1ba675678c9,41485,1731042802970, table=hbase:meta, region=1588230740 2024-11-08T05:13:30,736 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d1ba675678c9,41485,1731042802970, state=OPEN 2024-11-08T05:13:30,744 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41485-0x10118dcefab0002, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T05:13:30,744 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37587-0x10118dcefab0000, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T05:13:30,744 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39291-0x10118dcefab0003, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T05:13:30,745 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T05:13:30,745 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T05:13:30,745 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T05:13:30,746 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731042840746 2024-11-08T05:13:30,746 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-08T05:13:30,746 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure e61dc9e059921fddc5ad7f6f7971acc6, server=d1ba675678c9,41485,1731042802970:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d1ba675678c9,46753,1731042802851, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T05:13:30,748 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-08T05:13:30,750 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6240e38715889f3bb77bb89478c81a6c, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,751 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eeaad0e7e7aa66f0b51c273c0aaf953c, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,751 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=68bbd0353f97ee12228f7798b0b987ea, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,751 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=57, state=ROLLEDBACK, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure e61dc9e059921fddc5ad7f6f7971acc6, server=d1ba675678c9,41485,1731042802970:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d1ba675678c9,46753,1731042802851, splitWal=true, meta=true exec-time=2.6430 sec 2024-11-08T05:13:30,751 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=64ddf686fed30cdb23b2e874efec598d, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,751 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d7680a9929f15640ca61551f186f33c1, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,751 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=65d5312dbd2f96bf3b16bdfd7504dff6, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,751 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9b3274d80dcb7deed617c19ac1de4a68, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,751 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2d23e3e11ecc480e314e989d1bb8d0c3, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,751 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fe1545b16a81758dcb9430dbf98c4ab9, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,751 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a12ea80f78b339cfbfc0e18b5338df0c, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,752 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e1c63297d3a8aea2fdbd1842301c811, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,752 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=40440b5f39bb5421276aa0b156481a9e, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,752 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=27c9aaf34aaf98cd5bc8030630ae7fe3, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,752 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=868527ec322eba1d7cbd6371701f7ad2, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,752 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d951febea78018bb9f10f07d8788575d, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,752 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b0ebd9a9794acc56e430b36d1f17621c, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,752 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7830dc6b1acea6e1d468f8008d8a82b9, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,752 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12f1148a7b8db3e4f515337b8b7746fa, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,752 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e26dcb2339b364f3699d110c15cd4bb3, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,752 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=445b14856951586918c91d4ba342f1ff, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,752 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=14efcc1b9822cc74f4d4a96648555167, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,752 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7c8f72849b2e1719ed903afc73e0571a, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,752 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=81806cb184c70c9dc0df408694092d85, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,753 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=87b3455cffd1f4e06e5901b3bcbed9dc, regionState=OPEN, lastHost=d1ba675678c9,39291,1731042803020, regionLocation=d1ba675678c9,39291,1731042803020, openSeqNum=2 2024-11-08T05:13:30,753 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=de358f808110a48fa32e0533fefdc95c, regionState=OPEN, lastHost=d1ba675678c9,41485,1731042802970, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,753 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e61dc9e059921fddc5ad7f6f7971acc6, regionState=OPENING, lastHost=d1ba675678c9,46753,1731042802851, regionLocation=d1ba675678c9,41485,1731042802970, openSeqNum=2 2024-11-08T05:13:30,753 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-08T05:13:30,753 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731042870753 2024-11-08T05:13:30,753 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731042930753 2024-11-08T05:13:30,753 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 7 msec 2024-11-08T05:13:30,769 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: master.assignment.TestRollbackSCP#testFailAndRollback Thread=349 (was 374), OpenFileDescriptor=602 (was 616), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=271 (was 271), ProcessCount=11 (was 11), AvailableMemoryMB=7129 (was 7190) 2024-11-08T05:13:30,769 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-08T05:13:30,769 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-08T05:13:30,769 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.tearDownAfterClass(TestRollbackSCP.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T05:13:30,769 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T05:13:30,770 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T05:13:30,770 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T05:13:30,770 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-08T05:13:30,770 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1484730631, stopped=false 2024-11-08T05:13:30,770 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=d1ba675678c9,37587,1731042801796 2024-11-08T05:13:30,783 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41485-0x10118dcefab0002, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-08T05:13:30,783 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39291-0x10118dcefab0003, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-08T05:13:30,783 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37587-0x10118dcefab0000, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-08T05:13:30,783 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T05:13:30,783 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39291-0x10118dcefab0003, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T05:13:30,783 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37587-0x10118dcefab0000, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T05:13:30,783 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41485-0x10118dcefab0002, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T05:13:30,783 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-08T05:13:30,783 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.tearDownAfterClass(TestRollbackSCP.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T05:13:30,783 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T05:13:30,784 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'd1ba675678c9,41485,1731042802970' ***** 2024-11-08T05:13:30,784 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-08T05:13:30,784 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:39291-0x10118dcefab0003, quorum=127.0.0.1:57355, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T05:13:30,784 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'd1ba675678c9,39291,1731042803020' ***** 2024-11-08T05:13:30,784 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:37587-0x10118dcefab0000, quorum=127.0.0.1:57355, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T05:13:30,784 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-08T05:13:30,784 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:41485-0x10118dcefab0002, quorum=127.0.0.1:57355, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T05:13:30,784 INFO [RS:2;d1ba675678c9:39291 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-08T05:13:30,784 INFO [RS:1;d1ba675678c9:41485 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-08T05:13:30,784 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-08T05:13:30,784 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-08T05:13:30,784 INFO [RS:2;d1ba675678c9:39291 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-08T05:13:30,784 INFO [RS:2;d1ba675678c9:39291 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-08T05:13:30,784 INFO [RS:1;d1ba675678c9:41485 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-08T05:13:30,784 INFO [RS:1;d1ba675678c9:41485 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-08T05:13:30,785 INFO [RS:2;d1ba675678c9:39291 {}] regionserver.HRegionServer(3091): Received CLOSE for 14efcc1b9822cc74f4d4a96648555167 2024-11-08T05:13:30,785 INFO [RS:1;d1ba675678c9:41485 {}] regionserver.HRegionServer(3091): Received CLOSE for b0ebd9a9794acc56e430b36d1f17621c 2024-11-08T05:13:30,785 INFO [RS:2;d1ba675678c9:39291 {}] regionserver.HRegionServer(3091): Received CLOSE for 87b3455cffd1f4e06e5901b3bcbed9dc 2024-11-08T05:13:30,785 INFO [RS:1;d1ba675678c9:41485 {}] regionserver.HRegionServer(3091): Received CLOSE for 445b14856951586918c91d4ba342f1ff 2024-11-08T05:13:30,785 INFO [RS:2;d1ba675678c9:39291 {}] regionserver.HRegionServer(3091): Received CLOSE for 68bbd0353f97ee12228f7798b0b987ea 2024-11-08T05:13:30,785 INFO [RS:2;d1ba675678c9:39291 {}] regionserver.HRegionServer(3091): Received CLOSE for 12f1148a7b8db3e4f515337b8b7746fa 2024-11-08T05:13:30,785 INFO [RS:1;d1ba675678c9:41485 {}] regionserver.HRegionServer(3091): Received CLOSE for 65d5312dbd2f96bf3b16bdfd7504dff6 2024-11-08T05:13:30,785 INFO [RS:2;d1ba675678c9:39291 {}] regionserver.HRegionServer(3091): Received CLOSE for 6240e38715889f3bb77bb89478c81a6c 2024-11-08T05:13:30,785 INFO [RS:1;d1ba675678c9:41485 {}] regionserver.HRegionServer(3091): Received CLOSE for fe1545b16a81758dcb9430dbf98c4ab9 2024-11-08T05:13:30,785 INFO [RS:2;d1ba675678c9:39291 {}] regionserver.HRegionServer(3091): Received CLOSE for d7680a9929f15640ca61551f186f33c1 2024-11-08T05:13:30,785 INFO [RS:2;d1ba675678c9:39291 {}] regionserver.HRegionServer(3091): Received CLOSE for d951febea78018bb9f10f07d8788575d 2024-11-08T05:13:30,785 INFO [RS:1;d1ba675678c9:41485 {}] regionserver.HRegionServer(3091): Received CLOSE for e26dcb2339b364f3699d110c15cd4bb3 2024-11-08T05:13:30,785 INFO [RS:2;d1ba675678c9:39291 {}] regionserver.HRegionServer(3091): Received CLOSE for 2d23e3e11ecc480e314e989d1bb8d0c3 2024-11-08T05:13:30,785 INFO [RS:2;d1ba675678c9:39291 {}] regionserver.HRegionServer(3091): Received CLOSE for 40440b5f39bb5421276aa0b156481a9e 2024-11-08T05:13:30,785 INFO [RS:1;d1ba675678c9:41485 {}] regionserver.HRegionServer(3091): Received CLOSE for 0e1c63297d3a8aea2fdbd1842301c811 2024-11-08T05:13:30,786 INFO [RS:1;d1ba675678c9:41485 {}] regionserver.HRegionServer(3091): Received CLOSE for 7c8f72849b2e1719ed903afc73e0571a 2024-11-08T05:13:30,786 INFO [RS:2;d1ba675678c9:39291 {}] regionserver.HRegionServer(959): stopping server d1ba675678c9,39291,1731042803020 2024-11-08T05:13:30,786 INFO [RS:1;d1ba675678c9:41485 {}] regionserver.HRegionServer(3091): Received CLOSE for eeaad0e7e7aa66f0b51c273c0aaf953c 2024-11-08T05:13:30,786 INFO [RS:2;d1ba675678c9:39291 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-08T05:13:30,786 INFO [RS:1;d1ba675678c9:41485 {}] regionserver.HRegionServer(3091): Received CLOSE for e61dc9e059921fddc5ad7f6f7971acc6 2024-11-08T05:13:30,786 INFO [RS:2;d1ba675678c9:39291 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;d1ba675678c9:39291. 2024-11-08T05:13:30,786 INFO [RS:1;d1ba675678c9:41485 {}] regionserver.HRegionServer(3091): Received CLOSE for de358f808110a48fa32e0533fefdc95c 2024-11-08T05:13:30,786 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing b0ebd9a9794acc56e430b36d1f17621c, disabling compactions & flushes 2024-11-08T05:13:30,786 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 14efcc1b9822cc74f4d4a96648555167, disabling compactions & flushes 2024-11-08T05:13:30,786 INFO [RS:1;d1ba675678c9:41485 {}] regionserver.HRegionServer(3091): Received CLOSE for 27c9aaf34aaf98cd5bc8030630ae7fe3 2024-11-08T05:13:30,786 DEBUG [RS:2;d1ba675678c9:39291 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T05:13:30,786 DEBUG [RS:2;d1ba675678c9:39291 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T05:13:30,786 INFO [RS:1;d1ba675678c9:41485 {}] regionserver.HRegionServer(3091): Received CLOSE for 9b3274d80dcb7deed617c19ac1de4a68 2024-11-08T05:13:30,786 INFO [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,uuu,1731042805591.14efcc1b9822cc74f4d4a96648555167. 2024-11-08T05:13:30,786 INFO [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ppp,1731042805591.b0ebd9a9794acc56e430b36d1f17621c. 2024-11-08T05:13:30,786 INFO [RS:1;d1ba675678c9:41485 {}] regionserver.HRegionServer(3091): Received CLOSE for 868527ec322eba1d7cbd6371701f7ad2 2024-11-08T05:13:30,786 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,uuu,1731042805591.14efcc1b9822cc74f4d4a96648555167. 2024-11-08T05:13:30,786 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ppp,1731042805591.b0ebd9a9794acc56e430b36d1f17621c. 2024-11-08T05:13:30,786 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ppp,1731042805591.b0ebd9a9794acc56e430b36d1f17621c. after waiting 0 ms 2024-11-08T05:13:30,786 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,uuu,1731042805591.14efcc1b9822cc74f4d4a96648555167. after waiting 0 ms 2024-11-08T05:13:30,786 INFO [RS:1;d1ba675678c9:41485 {}] regionserver.HRegionServer(959): stopping server d1ba675678c9,41485,1731042802970 2024-11-08T05:13:30,786 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ppp,1731042805591.b0ebd9a9794acc56e430b36d1f17621c. 2024-11-08T05:13:30,786 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,uuu,1731042805591.14efcc1b9822cc74f4d4a96648555167. 2024-11-08T05:13:30,786 INFO [RS:1;d1ba675678c9:41485 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-08T05:13:30,786 INFO [RS:2;d1ba675678c9:39291 {}] regionserver.HRegionServer(1321): Waiting on 9 regions to close 2024-11-08T05:13:30,786 DEBUG [RS:2;d1ba675678c9:39291 {}] regionserver.HRegionServer(1325): Online Regions={14efcc1b9822cc74f4d4a96648555167=test,uuu,1731042805591.14efcc1b9822cc74f4d4a96648555167., 87b3455cffd1f4e06e5901b3bcbed9dc=test,xxx,1731042805591.87b3455cffd1f4e06e5901b3bcbed9dc., 68bbd0353f97ee12228f7798b0b987ea=test,ccc,1731042805591.68bbd0353f97ee12228f7798b0b987ea., 12f1148a7b8db3e4f515337b8b7746fa=test,rrr,1731042805591.12f1148a7b8db3e4f515337b8b7746fa., 6240e38715889f3bb77bb89478c81a6c=test,,1731042805591.6240e38715889f3bb77bb89478c81a6c., d7680a9929f15640ca61551f186f33c1=test,eee,1731042805591.d7680a9929f15640ca61551f186f33c1., d951febea78018bb9f10f07d8788575d=test,ooo,1731042805591.d951febea78018bb9f10f07d8788575d., 2d23e3e11ecc480e314e989d1bb8d0c3=test,hhh,1731042805591.2d23e3e11ecc480e314e989d1bb8d0c3., 40440b5f39bb5421276aa0b156481a9e=test,lll,1731042805591.40440b5f39bb5421276aa0b156481a9e.} 2024-11-08T05:13:30,786 INFO [RS:1;d1ba675678c9:41485 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;d1ba675678c9:41485. 2024-11-08T05:13:30,786 DEBUG [RS:1;d1ba675678c9:41485 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T05:13:30,787 DEBUG [RS:1;d1ba675678c9:41485 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T05:13:30,787 DEBUG [RS:2;d1ba675678c9:39291 {}] regionserver.HRegionServer(1351): Waiting on 12f1148a7b8db3e4f515337b8b7746fa, 14efcc1b9822cc74f4d4a96648555167, 2d23e3e11ecc480e314e989d1bb8d0c3, 40440b5f39bb5421276aa0b156481a9e, 6240e38715889f3bb77bb89478c81a6c, 64ddf686fed30cdb23b2e874efec598d, 68bbd0353f97ee12228f7798b0b987ea, 87b3455cffd1f4e06e5901b3bcbed9dc, d7680a9929f15640ca61551f186f33c1, d951febea78018bb9f10f07d8788575d 2024-11-08T05:13:30,787 INFO [RS:1;d1ba675678c9:41485 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-08T05:13:30,787 INFO [RS:1;d1ba675678c9:41485 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-08T05:13:30,787 INFO [RS:1;d1ba675678c9:41485 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-08T05:13:30,787 INFO [RS:1;d1ba675678c9:41485 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-08T05:13:30,788 INFO [RS:1;d1ba675678c9:41485 {}] regionserver.HRegionServer(1321): Waiting on 14 regions to close 2024-11-08T05:13:30,788 DEBUG [RS:1;d1ba675678c9:41485 {}] regionserver.HRegionServer(1351): Waiting on 0e1c63297d3a8aea2fdbd1842301c811, 1588230740, 27c9aaf34aaf98cd5bc8030630ae7fe3, 445b14856951586918c91d4ba342f1ff, 65d5312dbd2f96bf3b16bdfd7504dff6, 7c8f72849b2e1719ed903afc73e0571a, 868527ec322eba1d7cbd6371701f7ad2, 9b3274d80dcb7deed617c19ac1de4a68, b0ebd9a9794acc56e430b36d1f17621c, de358f808110a48fa32e0533fefdc95c, e26dcb2339b364f3699d110c15cd4bb3, e61dc9e059921fddc5ad7f6f7971acc6, eeaad0e7e7aa66f0b51c273c0aaf953c, fe1545b16a81758dcb9430dbf98c4ab9 2024-11-08T05:13:30,788 DEBUG [RS_CLOSE_META-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-08T05:13:30,788 INFO [RS_CLOSE_META-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-08T05:13:30,788 DEBUG [RS_CLOSE_META-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-08T05:13:30,788 DEBUG [RS_CLOSE_META-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-08T05:13:30,788 DEBUG [RS_CLOSE_META-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-08T05:13:30,789 INFO [RS_CLOSE_META-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=2.65 KB heapSize=5.44 KB 2024-11-08T05:13:30,792 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/default/test/b0ebd9a9794acc56e430b36d1f17621c/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-08T05:13:30,792 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/default/test/14efcc1b9822cc74f4d4a96648555167/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-08T05:13:30,794 INFO [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ppp,1731042805591.b0ebd9a9794acc56e430b36d1f17621c. 2024-11-08T05:13:30,794 INFO [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,uuu,1731042805591.14efcc1b9822cc74f4d4a96648555167. 2024-11-08T05:13:30,794 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for b0ebd9a9794acc56e430b36d1f17621c: Waiting for close lock at 1731042810785Running coprocessor pre-close hooks at 1731042810786 (+1 ms)Disabling compacts and flushes for region at 1731042810786Disabling writes for close at 1731042810786Writing region close event to WAL at 1731042810788 (+2 ms)Running coprocessor post-close hooks at 1731042810793 (+5 ms)Closed at 1731042810794 (+1 ms) 2024-11-08T05:13:30,794 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 14efcc1b9822cc74f4d4a96648555167: Waiting for close lock at 1731042810785Running coprocessor pre-close hooks at 1731042810786 (+1 ms)Disabling compacts and flushes for region at 1731042810786Disabling writes for close at 1731042810786Writing region close event to WAL at 1731042810787 (+1 ms)Running coprocessor post-close hooks at 1731042810793 (+6 ms)Closed at 1731042810794 (+1 ms) 2024-11-08T05:13:30,795 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ppp,1731042805591.b0ebd9a9794acc56e430b36d1f17621c. 2024-11-08T05:13:30,795 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,uuu,1731042805591.14efcc1b9822cc74f4d4a96648555167. 2024-11-08T05:13:30,795 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 445b14856951586918c91d4ba342f1ff, disabling compactions & flushes 2024-11-08T05:13:30,795 INFO [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ttt,1731042805591.445b14856951586918c91d4ba342f1ff. 2024-11-08T05:13:30,795 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 87b3455cffd1f4e06e5901b3bcbed9dc, disabling compactions & flushes 2024-11-08T05:13:30,795 INFO [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,xxx,1731042805591.87b3455cffd1f4e06e5901b3bcbed9dc. 2024-11-08T05:13:30,795 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ttt,1731042805591.445b14856951586918c91d4ba342f1ff. 2024-11-08T05:13:30,795 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ttt,1731042805591.445b14856951586918c91d4ba342f1ff. after waiting 0 ms 2024-11-08T05:13:30,795 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,xxx,1731042805591.87b3455cffd1f4e06e5901b3bcbed9dc. 2024-11-08T05:13:30,795 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ttt,1731042805591.445b14856951586918c91d4ba342f1ff. 2024-11-08T05:13:30,795 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,xxx,1731042805591.87b3455cffd1f4e06e5901b3bcbed9dc. after waiting 0 ms 2024-11-08T05:13:30,795 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,xxx,1731042805591.87b3455cffd1f4e06e5901b3bcbed9dc. 2024-11-08T05:13:30,801 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/default/test/87b3455cffd1f4e06e5901b3bcbed9dc/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-08T05:13:30,802 INFO [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,xxx,1731042805591.87b3455cffd1f4e06e5901b3bcbed9dc. 2024-11-08T05:13:30,802 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 87b3455cffd1f4e06e5901b3bcbed9dc: Waiting for close lock at 1731042810795Running coprocessor pre-close hooks at 1731042810795Disabling compacts and flushes for region at 1731042810795Disabling writes for close at 1731042810795Writing region close event to WAL at 1731042810796 (+1 ms)Running coprocessor post-close hooks at 1731042810801 (+5 ms)Closed at 1731042810802 (+1 ms) 2024-11-08T05:13:30,802 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,xxx,1731042805591.87b3455cffd1f4e06e5901b3bcbed9dc. 2024-11-08T05:13:30,802 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 68bbd0353f97ee12228f7798b0b987ea, disabling compactions & flushes 2024-11-08T05:13:30,802 INFO [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ccc,1731042805591.68bbd0353f97ee12228f7798b0b987ea. 2024-11-08T05:13:30,802 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ccc,1731042805591.68bbd0353f97ee12228f7798b0b987ea. 2024-11-08T05:13:30,802 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ccc,1731042805591.68bbd0353f97ee12228f7798b0b987ea. after waiting 0 ms 2024-11-08T05:13:30,802 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ccc,1731042805591.68bbd0353f97ee12228f7798b0b987ea. 2024-11-08T05:13:30,802 DEBUG [RS_CLOSE_META-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/hbase/meta/1588230740/.tmp/info/7cc0eba8728a48f7b2bfd9e27e6f44a2 is 138, key is test,ddd,1731042805591.64ddf686fed30cdb23b2e874efec598d./info:regioninfo/1731042809625/Put/seqid=0 2024-11-08T05:13:30,803 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/default/test/445b14856951586918c91d4ba342f1ff/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-08T05:13:30,804 INFO [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ttt,1731042805591.445b14856951586918c91d4ba342f1ff. 2024-11-08T05:13:30,804 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 445b14856951586918c91d4ba342f1ff: Waiting for close lock at 1731042810795Running coprocessor pre-close hooks at 1731042810795Disabling compacts and flushes for region at 1731042810795Disabling writes for close at 1731042810795Writing region close event to WAL at 1731042810796 (+1 ms)Running coprocessor post-close hooks at 1731042810804 (+8 ms)Closed at 1731042810804 2024-11-08T05:13:30,804 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ttt,1731042805591.445b14856951586918c91d4ba342f1ff. 2024-11-08T05:13:30,804 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 65d5312dbd2f96bf3b16bdfd7504dff6, disabling compactions & flushes 2024-11-08T05:13:30,804 INFO [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,fff,1731042805591.65d5312dbd2f96bf3b16bdfd7504dff6. 2024-11-08T05:13:30,804 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,fff,1731042805591.65d5312dbd2f96bf3b16bdfd7504dff6. 2024-11-08T05:13:30,804 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,fff,1731042805591.65d5312dbd2f96bf3b16bdfd7504dff6. after waiting 0 ms 2024-11-08T05:13:30,804 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,fff,1731042805591.65d5312dbd2f96bf3b16bdfd7504dff6. 2024-11-08T05:13:30,807 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/default/test/68bbd0353f97ee12228f7798b0b987ea/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-08T05:13:30,807 INFO [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ccc,1731042805591.68bbd0353f97ee12228f7798b0b987ea. 2024-11-08T05:13:30,808 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 68bbd0353f97ee12228f7798b0b987ea: Waiting for close lock at 1731042810802Running coprocessor pre-close hooks at 1731042810802Disabling compacts and flushes for region at 1731042810802Disabling writes for close at 1731042810802Writing region close event to WAL at 1731042810803 (+1 ms)Running coprocessor post-close hooks at 1731042810807 (+4 ms)Closed at 1731042810807 2024-11-08T05:13:30,808 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ccc,1731042805591.68bbd0353f97ee12228f7798b0b987ea. 2024-11-08T05:13:30,808 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 12f1148a7b8db3e4f515337b8b7746fa, disabling compactions & flushes 2024-11-08T05:13:30,808 INFO [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,rrr,1731042805591.12f1148a7b8db3e4f515337b8b7746fa. 2024-11-08T05:13:30,808 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,rrr,1731042805591.12f1148a7b8db3e4f515337b8b7746fa. 2024-11-08T05:13:30,808 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,rrr,1731042805591.12f1148a7b8db3e4f515337b8b7746fa. after waiting 0 ms 2024-11-08T05:13:30,808 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,rrr,1731042805591.12f1148a7b8db3e4f515337b8b7746fa. 2024-11-08T05:13:30,813 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/default/test/65d5312dbd2f96bf3b16bdfd7504dff6/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-08T05:13:30,814 INFO [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,fff,1731042805591.65d5312dbd2f96bf3b16bdfd7504dff6. 2024-11-08T05:13:30,814 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 65d5312dbd2f96bf3b16bdfd7504dff6: Waiting for close lock at 1731042810804Running coprocessor pre-close hooks at 1731042810804Disabling compacts and flushes for region at 1731042810804Disabling writes for close at 1731042810804Writing region close event to WAL at 1731042810805 (+1 ms)Running coprocessor post-close hooks at 1731042810814 (+9 ms)Closed at 1731042810814 2024-11-08T05:13:30,814 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,fff,1731042805591.65d5312dbd2f96bf3b16bdfd7504dff6. 2024-11-08T05:13:30,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741869_1045 (size=8192) 2024-11-08T05:13:30,815 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing fe1545b16a81758dcb9430dbf98c4ab9, disabling compactions & flushes 2024-11-08T05:13:30,815 INFO [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,iii,1731042805591.fe1545b16a81758dcb9430dbf98c4ab9. 2024-11-08T05:13:30,815 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,iii,1731042805591.fe1545b16a81758dcb9430dbf98c4ab9. 2024-11-08T05:13:30,815 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,iii,1731042805591.fe1545b16a81758dcb9430dbf98c4ab9. after waiting 0 ms 2024-11-08T05:13:30,815 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,iii,1731042805591.fe1545b16a81758dcb9430dbf98c4ab9. 2024-11-08T05:13:30,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741869_1045 (size=8192) 2024-11-08T05:13:30,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34565 is added to blk_1073741869_1045 (size=8192) 2024-11-08T05:13:30,816 INFO [RS_CLOSE_META-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.65 KB at sequenceid=71 (bloomFilter=true), to=hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/hbase/meta/1588230740/.tmp/info/7cc0eba8728a48f7b2bfd9e27e6f44a2 2024-11-08T05:13:30,818 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/default/test/12f1148a7b8db3e4f515337b8b7746fa/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-08T05:13:30,819 INFO [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,rrr,1731042805591.12f1148a7b8db3e4f515337b8b7746fa. 2024-11-08T05:13:30,819 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 12f1148a7b8db3e4f515337b8b7746fa: Waiting for close lock at 1731042810808Running coprocessor pre-close hooks at 1731042810808Disabling compacts and flushes for region at 1731042810808Disabling writes for close at 1731042810808Writing region close event to WAL at 1731042810812 (+4 ms)Running coprocessor post-close hooks at 1731042810819 (+7 ms)Closed at 1731042810819 2024-11-08T05:13:30,819 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,rrr,1731042805591.12f1148a7b8db3e4f515337b8b7746fa. 2024-11-08T05:13:30,820 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 6240e38715889f3bb77bb89478c81a6c, disabling compactions & flushes 2024-11-08T05:13:30,820 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/default/test/fe1545b16a81758dcb9430dbf98c4ab9/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-08T05:13:30,820 INFO [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,,1731042805591.6240e38715889f3bb77bb89478c81a6c. 2024-11-08T05:13:30,820 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,,1731042805591.6240e38715889f3bb77bb89478c81a6c. 2024-11-08T05:13:30,820 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,,1731042805591.6240e38715889f3bb77bb89478c81a6c. after waiting 0 ms 2024-11-08T05:13:30,820 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,,1731042805591.6240e38715889f3bb77bb89478c81a6c. 2024-11-08T05:13:30,820 INFO [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,iii,1731042805591.fe1545b16a81758dcb9430dbf98c4ab9. 2024-11-08T05:13:30,820 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for fe1545b16a81758dcb9430dbf98c4ab9: Waiting for close lock at 1731042810815Running coprocessor pre-close hooks at 1731042810815Disabling compacts and flushes for region at 1731042810815Disabling writes for close at 1731042810815Writing region close event to WAL at 1731042810815Running coprocessor post-close hooks at 1731042810820 (+5 ms)Closed at 1731042810820 2024-11-08T05:13:30,820 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,iii,1731042805591.fe1545b16a81758dcb9430dbf98c4ab9. 2024-11-08T05:13:30,821 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing e26dcb2339b364f3699d110c15cd4bb3, disabling compactions & flushes 2024-11-08T05:13:30,821 INFO [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,sss,1731042805591.e26dcb2339b364f3699d110c15cd4bb3. 2024-11-08T05:13:30,821 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,sss,1731042805591.e26dcb2339b364f3699d110c15cd4bb3. 2024-11-08T05:13:30,821 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,sss,1731042805591.e26dcb2339b364f3699d110c15cd4bb3. after waiting 0 ms 2024-11-08T05:13:30,821 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,sss,1731042805591.e26dcb2339b364f3699d110c15cd4bb3. 2024-11-08T05:13:30,825 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/default/test/e26dcb2339b364f3699d110c15cd4bb3/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-08T05:13:30,825 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/default/test/6240e38715889f3bb77bb89478c81a6c/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-08T05:13:30,826 INFO [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,sss,1731042805591.e26dcb2339b364f3699d110c15cd4bb3. 2024-11-08T05:13:30,826 DEBUG [RS_CLOSE_META-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/hbase/meta/1588230740/.tmp/info/7cc0eba8728a48f7b2bfd9e27e6f44a2 as hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/hbase/meta/1588230740/info/7cc0eba8728a48f7b2bfd9e27e6f44a2 2024-11-08T05:13:30,826 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for e26dcb2339b364f3699d110c15cd4bb3: Waiting for close lock at 1731042810821Running coprocessor pre-close hooks at 1731042810821Disabling compacts and flushes for region at 1731042810821Disabling writes for close at 1731042810821Writing region close event to WAL at 1731042810821Running coprocessor post-close hooks at 1731042810825 (+4 ms)Closed at 1731042810826 (+1 ms) 2024-11-08T05:13:30,826 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,sss,1731042805591.e26dcb2339b364f3699d110c15cd4bb3. 2024-11-08T05:13:30,826 INFO [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,,1731042805591.6240e38715889f3bb77bb89478c81a6c. 2024-11-08T05:13:30,826 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 6240e38715889f3bb77bb89478c81a6c: Waiting for close lock at 1731042810820Running coprocessor pre-close hooks at 1731042810820Disabling compacts and flushes for region at 1731042810820Disabling writes for close at 1731042810820Writing region close event to WAL at 1731042810820Running coprocessor post-close hooks at 1731042810826 (+6 ms)Closed at 1731042810826 2024-11-08T05:13:30,826 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 0e1c63297d3a8aea2fdbd1842301c811, disabling compactions & flushes 2024-11-08T05:13:30,826 INFO [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,kkk,1731042805591.0e1c63297d3a8aea2fdbd1842301c811. 2024-11-08T05:13:30,826 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,,1731042805591.6240e38715889f3bb77bb89478c81a6c. 2024-11-08T05:13:30,826 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,kkk,1731042805591.0e1c63297d3a8aea2fdbd1842301c811. 2024-11-08T05:13:30,826 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,kkk,1731042805591.0e1c63297d3a8aea2fdbd1842301c811. after waiting 0 ms 2024-11-08T05:13:30,826 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,kkk,1731042805591.0e1c63297d3a8aea2fdbd1842301c811. 2024-11-08T05:13:30,826 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing d7680a9929f15640ca61551f186f33c1, disabling compactions & flushes 2024-11-08T05:13:30,826 INFO [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,eee,1731042805591.d7680a9929f15640ca61551f186f33c1. 2024-11-08T05:13:30,826 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,eee,1731042805591.d7680a9929f15640ca61551f186f33c1. 2024-11-08T05:13:30,826 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,eee,1731042805591.d7680a9929f15640ca61551f186f33c1. after waiting 0 ms 2024-11-08T05:13:30,826 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,eee,1731042805591.d7680a9929f15640ca61551f186f33c1. 2024-11-08T05:13:30,831 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/default/test/0e1c63297d3a8aea2fdbd1842301c811/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-08T05:13:30,832 INFO [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,kkk,1731042805591.0e1c63297d3a8aea2fdbd1842301c811. 2024-11-08T05:13:30,832 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 0e1c63297d3a8aea2fdbd1842301c811: Waiting for close lock at 1731042810826Running coprocessor pre-close hooks at 1731042810826Disabling compacts and flushes for region at 1731042810826Disabling writes for close at 1731042810826Writing region close event to WAL at 1731042810827 (+1 ms)Running coprocessor post-close hooks at 1731042810832 (+5 ms)Closed at 1731042810832 2024-11-08T05:13:30,833 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,kkk,1731042805591.0e1c63297d3a8aea2fdbd1842301c811. 2024-11-08T05:13:30,833 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 7c8f72849b2e1719ed903afc73e0571a, disabling compactions & flushes 2024-11-08T05:13:30,833 INFO [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,vvv,1731042805591.7c8f72849b2e1719ed903afc73e0571a. 2024-11-08T05:13:30,833 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,vvv,1731042805591.7c8f72849b2e1719ed903afc73e0571a. 2024-11-08T05:13:30,833 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,vvv,1731042805591.7c8f72849b2e1719ed903afc73e0571a. after waiting 0 ms 2024-11-08T05:13:30,833 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,vvv,1731042805591.7c8f72849b2e1719ed903afc73e0571a. 2024-11-08T05:13:30,834 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/default/test/d7680a9929f15640ca61551f186f33c1/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-08T05:13:30,835 INFO [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,eee,1731042805591.d7680a9929f15640ca61551f186f33c1. 2024-11-08T05:13:30,835 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for d7680a9929f15640ca61551f186f33c1: Waiting for close lock at 1731042810826Running coprocessor pre-close hooks at 1731042810826Disabling compacts and flushes for region at 1731042810826Disabling writes for close at 1731042810826Writing region close event to WAL at 1731042810827 (+1 ms)Running coprocessor post-close hooks at 1731042810835 (+8 ms)Closed at 1731042810835 2024-11-08T05:13:30,835 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,eee,1731042805591.d7680a9929f15640ca61551f186f33c1. 2024-11-08T05:13:30,836 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing d951febea78018bb9f10f07d8788575d, disabling compactions & flushes 2024-11-08T05:13:30,836 INFO [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ooo,1731042805591.d951febea78018bb9f10f07d8788575d. 2024-11-08T05:13:30,836 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ooo,1731042805591.d951febea78018bb9f10f07d8788575d. 2024-11-08T05:13:30,836 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ooo,1731042805591.d951febea78018bb9f10f07d8788575d. after waiting 0 ms 2024-11-08T05:13:30,836 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ooo,1731042805591.d951febea78018bb9f10f07d8788575d. 2024-11-08T05:13:30,837 INFO [RS_CLOSE_META-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/hbase/meta/1588230740/info/7cc0eba8728a48f7b2bfd9e27e6f44a2, entries=24, sequenceid=71, filesize=8 K 2024-11-08T05:13:30,838 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/default/test/7c8f72849b2e1719ed903afc73e0571a/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-08T05:13:30,839 INFO [regionserver/d1ba675678c9:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-08T05:13:30,839 INFO [RS_CLOSE_META-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~2.65 KB/2717, heapSize ~4.67 KB/4784, currentSize=0 B/0 for 1588230740 in 50ms, sequenceid=71, compaction requested=false 2024-11-08T05:13:30,839 INFO [regionserver/d1ba675678c9:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-08T05:13:30,839 INFO [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,vvv,1731042805591.7c8f72849b2e1719ed903afc73e0571a. 2024-11-08T05:13:30,839 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 7c8f72849b2e1719ed903afc73e0571a: Waiting for close lock at 1731042810833Running coprocessor pre-close hooks at 1731042810833Disabling compacts and flushes for region at 1731042810833Disabling writes for close at 1731042810833Writing region close event to WAL at 1731042810833Running coprocessor post-close hooks at 1731042810839 (+6 ms)Closed at 1731042810839 2024-11-08T05:13:30,839 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,vvv,1731042805591.7c8f72849b2e1719ed903afc73e0571a. 2024-11-08T05:13:30,840 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing eeaad0e7e7aa66f0b51c273c0aaf953c, disabling compactions & flushes 2024-11-08T05:13:30,840 INFO [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,bbb,1731042805591.eeaad0e7e7aa66f0b51c273c0aaf953c. 2024-11-08T05:13:30,840 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,bbb,1731042805591.eeaad0e7e7aa66f0b51c273c0aaf953c. 2024-11-08T05:13:30,840 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,bbb,1731042805591.eeaad0e7e7aa66f0b51c273c0aaf953c. after waiting 0 ms 2024-11-08T05:13:30,840 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,bbb,1731042805591.eeaad0e7e7aa66f0b51c273c0aaf953c. 2024-11-08T05:13:30,845 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/default/test/eeaad0e7e7aa66f0b51c273c0aaf953c/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-08T05:13:30,845 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/default/test/d951febea78018bb9f10f07d8788575d/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-08T05:13:30,846 INFO [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,bbb,1731042805591.eeaad0e7e7aa66f0b51c273c0aaf953c. 2024-11-08T05:13:30,846 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for eeaad0e7e7aa66f0b51c273c0aaf953c: Waiting for close lock at 1731042810840Running coprocessor pre-close hooks at 1731042810840Disabling compacts and flushes for region at 1731042810840Disabling writes for close at 1731042810840Writing region close event to WAL at 1731042810841 (+1 ms)Running coprocessor post-close hooks at 1731042810846 (+5 ms)Closed at 1731042810846 2024-11-08T05:13:30,846 INFO [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ooo,1731042805591.d951febea78018bb9f10f07d8788575d. 2024-11-08T05:13:30,846 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for d951febea78018bb9f10f07d8788575d: Waiting for close lock at 1731042810836Running coprocessor pre-close hooks at 1731042810836Disabling compacts and flushes for region at 1731042810836Disabling writes for close at 1731042810836Writing region close event to WAL at 1731042810837 (+1 ms)Running coprocessor post-close hooks at 1731042810846 (+9 ms)Closed at 1731042810846 2024-11-08T05:13:30,846 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,bbb,1731042805591.eeaad0e7e7aa66f0b51c273c0aaf953c. 2024-11-08T05:13:30,846 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ooo,1731042805591.d951febea78018bb9f10f07d8788575d. 2024-11-08T05:13:30,846 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 2d23e3e11ecc480e314e989d1bb8d0c3, disabling compactions & flushes 2024-11-08T05:13:30,846 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing e61dc9e059921fddc5ad7f6f7971acc6, disabling compactions & flushes 2024-11-08T05:13:30,846 INFO [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,hhh,1731042805591.2d23e3e11ecc480e314e989d1bb8d0c3. 2024-11-08T05:13:30,846 INFO [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,zzz,1731042805591.e61dc9e059921fddc5ad7f6f7971acc6. 2024-11-08T05:13:30,846 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,hhh,1731042805591.2d23e3e11ecc480e314e989d1bb8d0c3. 2024-11-08T05:13:30,846 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,zzz,1731042805591.e61dc9e059921fddc5ad7f6f7971acc6. 2024-11-08T05:13:30,846 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,hhh,1731042805591.2d23e3e11ecc480e314e989d1bb8d0c3. after waiting 0 ms 2024-11-08T05:13:30,846 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,zzz,1731042805591.e61dc9e059921fddc5ad7f6f7971acc6. after waiting 0 ms 2024-11-08T05:13:30,847 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,hhh,1731042805591.2d23e3e11ecc480e314e989d1bb8d0c3. 2024-11-08T05:13:30,847 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,zzz,1731042805591.e61dc9e059921fddc5ad7f6f7971acc6. 2024-11-08T05:13:30,853 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/default/test/e61dc9e059921fddc5ad7f6f7971acc6/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-08T05:13:30,853 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/default/test/2d23e3e11ecc480e314e989d1bb8d0c3/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-08T05:13:30,853 DEBUG [RS_CLOSE_META-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/hbase/meta/1588230740/recovered.edits/74.seqid, newMaxSeqId=74, maxSeqId=60 2024-11-08T05:13:30,853 INFO [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,zzz,1731042805591.e61dc9e059921fddc5ad7f6f7971acc6. 2024-11-08T05:13:30,853 INFO [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,hhh,1731042805591.2d23e3e11ecc480e314e989d1bb8d0c3. 2024-11-08T05:13:30,853 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for e61dc9e059921fddc5ad7f6f7971acc6: Waiting for close lock at 1731042810846Running coprocessor pre-close hooks at 1731042810846Disabling compacts and flushes for region at 1731042810846Disabling writes for close at 1731042810847 (+1 ms)Writing region close event to WAL at 1731042810847Running coprocessor post-close hooks at 1731042810853 (+6 ms)Closed at 1731042810853 2024-11-08T05:13:30,854 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 2d23e3e11ecc480e314e989d1bb8d0c3: Waiting for close lock at 1731042810846Running coprocessor pre-close hooks at 1731042810846Disabling compacts and flushes for region at 1731042810846Disabling writes for close at 1731042810847 (+1 ms)Writing region close event to WAL at 1731042810847Running coprocessor post-close hooks at 1731042810853 (+6 ms)Closed at 1731042810853 2024-11-08T05:13:30,854 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,zzz,1731042805591.e61dc9e059921fddc5ad7f6f7971acc6. 2024-11-08T05:13:30,854 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,hhh,1731042805591.2d23e3e11ecc480e314e989d1bb8d0c3. 2024-11-08T05:13:30,854 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing de358f808110a48fa32e0533fefdc95c, disabling compactions & flushes 2024-11-08T05:13:30,854 DEBUG [RS_CLOSE_META-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-08T05:13:30,854 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 40440b5f39bb5421276aa0b156481a9e, disabling compactions & flushes 2024-11-08T05:13:30,854 INFO [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,yyy,1731042805591.de358f808110a48fa32e0533fefdc95c. 2024-11-08T05:13:30,854 INFO [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,lll,1731042805591.40440b5f39bb5421276aa0b156481a9e. 2024-11-08T05:13:30,854 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,yyy,1731042805591.de358f808110a48fa32e0533fefdc95c. 2024-11-08T05:13:30,854 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,yyy,1731042805591.de358f808110a48fa32e0533fefdc95c. after waiting 0 ms 2024-11-08T05:13:30,854 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,lll,1731042805591.40440b5f39bb5421276aa0b156481a9e. 2024-11-08T05:13:30,854 INFO [RS_CLOSE_META-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-08T05:13:30,854 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,lll,1731042805591.40440b5f39bb5421276aa0b156481a9e. after waiting 0 ms 2024-11-08T05:13:30,854 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,yyy,1731042805591.de358f808110a48fa32e0533fefdc95c. 2024-11-08T05:13:30,854 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,lll,1731042805591.40440b5f39bb5421276aa0b156481a9e. 2024-11-08T05:13:30,854 DEBUG [RS_CLOSE_META-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731042810788Running coprocessor pre-close hooks at 1731042810788Disabling compacts and flushes for region at 1731042810788Disabling writes for close at 1731042810788Obtaining lock to block concurrent updates at 1731042810789 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1731042810789Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=2717, getHeapSize=5504, getOffHeapSize=0, getCellsCount=24 at 1731042810796 (+7 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731042810798 (+2 ms)Flushing 1588230740/info: creating writer at 1731042810798Flushing 1588230740/info: appending metadata at 1731042810802 (+4 ms)Flushing 1588230740/info: closing flushed file at 1731042810802Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@767f1658: reopening flushed file at 1731042810825 (+23 ms)Finished flush of dataSize ~2.65 KB/2717, heapSize ~4.67 KB/4784, currentSize=0 B/0 for 1588230740 in 50ms, sequenceid=71, compaction requested=false at 1731042810839 (+14 ms)Writing region close event to WAL at 1731042810848 (+9 ms)Running coprocessor post-close hooks at 1731042810854 (+6 ms)Closed at 1731042810854 2024-11-08T05:13:30,854 DEBUG [RS_CLOSE_META-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-08T05:13:30,858 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/default/test/de358f808110a48fa32e0533fefdc95c/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-08T05:13:30,858 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/default/test/40440b5f39bb5421276aa0b156481a9e/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-08T05:13:30,859 INFO [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,lll,1731042805591.40440b5f39bb5421276aa0b156481a9e. 2024-11-08T05:13:30,859 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 40440b5f39bb5421276aa0b156481a9e: Waiting for close lock at 1731042810854Running coprocessor pre-close hooks at 1731042810854Disabling compacts and flushes for region at 1731042810854Disabling writes for close at 1731042810854Writing region close event to WAL at 1731042810854Running coprocessor post-close hooks at 1731042810859 (+5 ms)Closed at 1731042810859 2024-11-08T05:13:30,859 INFO [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,yyy,1731042805591.de358f808110a48fa32e0533fefdc95c. 2024-11-08T05:13:30,859 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for de358f808110a48fa32e0533fefdc95c: Waiting for close lock at 1731042810854Running coprocessor pre-close hooks at 1731042810854Disabling compacts and flushes for region at 1731042810854Disabling writes for close at 1731042810854Writing region close event to WAL at 1731042810854Running coprocessor post-close hooks at 1731042810859 (+5 ms)Closed at 1731042810859 2024-11-08T05:13:30,859 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,lll,1731042805591.40440b5f39bb5421276aa0b156481a9e. 2024-11-08T05:13:30,859 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,yyy,1731042805591.de358f808110a48fa32e0533fefdc95c. 2024-11-08T05:13:30,859 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 27c9aaf34aaf98cd5bc8030630ae7fe3, disabling compactions & flushes 2024-11-08T05:13:30,859 INFO [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,mmm,1731042805591.27c9aaf34aaf98cd5bc8030630ae7fe3. 2024-11-08T05:13:30,860 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,mmm,1731042805591.27c9aaf34aaf98cd5bc8030630ae7fe3. 2024-11-08T05:13:30,860 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,mmm,1731042805591.27c9aaf34aaf98cd5bc8030630ae7fe3. after waiting 0 ms 2024-11-08T05:13:30,860 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,mmm,1731042805591.27c9aaf34aaf98cd5bc8030630ae7fe3. 2024-11-08T05:13:30,865 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/default/test/27c9aaf34aaf98cd5bc8030630ae7fe3/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-08T05:13:30,866 INFO [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,mmm,1731042805591.27c9aaf34aaf98cd5bc8030630ae7fe3. 2024-11-08T05:13:30,866 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 27c9aaf34aaf98cd5bc8030630ae7fe3: Waiting for close lock at 1731042810859Running coprocessor pre-close hooks at 1731042810859Disabling compacts and flushes for region at 1731042810859Disabling writes for close at 1731042810860 (+1 ms)Writing region close event to WAL at 1731042810860Running coprocessor post-close hooks at 1731042810866 (+6 ms)Closed at 1731042810866 2024-11-08T05:13:30,866 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,mmm,1731042805591.27c9aaf34aaf98cd5bc8030630ae7fe3. 2024-11-08T05:13:30,866 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 9b3274d80dcb7deed617c19ac1de4a68, disabling compactions & flushes 2024-11-08T05:13:30,866 INFO [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ggg,1731042805591.9b3274d80dcb7deed617c19ac1de4a68. 2024-11-08T05:13:30,866 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ggg,1731042805591.9b3274d80dcb7deed617c19ac1de4a68. 2024-11-08T05:13:30,866 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ggg,1731042805591.9b3274d80dcb7deed617c19ac1de4a68. after waiting 0 ms 2024-11-08T05:13:30,866 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ggg,1731042805591.9b3274d80dcb7deed617c19ac1de4a68. 2024-11-08T05:13:30,871 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/default/test/9b3274d80dcb7deed617c19ac1de4a68/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-08T05:13:30,872 INFO [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ggg,1731042805591.9b3274d80dcb7deed617c19ac1de4a68. 2024-11-08T05:13:30,872 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 9b3274d80dcb7deed617c19ac1de4a68: Waiting for close lock at 1731042810866Running coprocessor pre-close hooks at 1731042810866Disabling compacts and flushes for region at 1731042810866Disabling writes for close at 1731042810866Writing region close event to WAL at 1731042810867 (+1 ms)Running coprocessor post-close hooks at 1731042810872 (+5 ms)Closed at 1731042810872 2024-11-08T05:13:30,872 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ggg,1731042805591.9b3274d80dcb7deed617c19ac1de4a68. 2024-11-08T05:13:30,872 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 868527ec322eba1d7cbd6371701f7ad2, disabling compactions & flushes 2024-11-08T05:13:30,873 INFO [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,nnn,1731042805591.868527ec322eba1d7cbd6371701f7ad2. 2024-11-08T05:13:30,873 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,nnn,1731042805591.868527ec322eba1d7cbd6371701f7ad2. 2024-11-08T05:13:30,873 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,nnn,1731042805591.868527ec322eba1d7cbd6371701f7ad2. after waiting 0 ms 2024-11-08T05:13:30,873 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,nnn,1731042805591.868527ec322eba1d7cbd6371701f7ad2. 2024-11-08T05:13:30,878 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/default/test/868527ec322eba1d7cbd6371701f7ad2/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-08T05:13:30,879 INFO [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,nnn,1731042805591.868527ec322eba1d7cbd6371701f7ad2. 2024-11-08T05:13:30,879 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 868527ec322eba1d7cbd6371701f7ad2: Waiting for close lock at 1731042810872Running coprocessor pre-close hooks at 1731042810872Disabling compacts and flushes for region at 1731042810872Disabling writes for close at 1731042810873 (+1 ms)Writing region close event to WAL at 1731042810874 (+1 ms)Running coprocessor post-close hooks at 1731042810879 (+5 ms)Closed at 1731042810879 2024-11-08T05:13:30,879 DEBUG [RS_CLOSE_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,nnn,1731042805591.868527ec322eba1d7cbd6371701f7ad2. 2024-11-08T05:13:30,924 WARN [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(168): Fatal error occurred while opening region test,ddd,1731042805591.64ddf686fed30cdb23b2e874efec598d., aborting... java.io.IOException: Failed to report opened region to master: test,ddd,1731042805591.64ddf686fed30cdb23b2e874efec598d. at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2261) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T05:13:30,925 ERROR [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2440): ***** ABORTING region server d1ba675678c9,39291,1731042803020: Failed to open region test,ddd,1731042805591.64ddf686fed30cdb23b2e874efec598d. and can not recover ***** java.io.IOException: Failed to report opened region to master: test,ddd,1731042805591.64ddf686fed30cdb23b2e874efec598d. at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2261) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T05:13:30,925 ERROR [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-08T05:13:30,925 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-08T05:13:30,926 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-08T05:13:30,930 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-08T05:13:30,932 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-08T05:13:30,941 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "Verbose": false, "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 471459328 }, "NonHeapMemoryUsage": { "committed": 162791424, "init": 7667712, "max": -1, "used": 160096992 }, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "d1ba675678c9", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 0, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 40413, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 130, "ProcessCallTime_min": 0, "ProcessCallTime_max": 26, "ProcessCallTime_mean": 2, "ProcessCallTime_25th_percentile": 6, "ProcessCallTime_median": 13, "ProcessCallTime_75th_percentile": 19, "ProcessCallTime_90th_percentile": 23, "ProcessCallTime_95th_percentile": 24, "ProcessCallTime_98th_percentile": 25, "ProcessCallTime_99th_percentile": 25, "ProcessCallTime_99.9th_percentile": 25, "ProcessCallTime_TimeRangeCount_0-1": 130, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 130, "QueueCallTime_min": 0, "QueueCallTime_max": 14, "QueueCallTime_mean": 1, "QueueCallTime_25th_percentile": 3, "QueueCallTime_median": 7, "QueueCallTime_75th_percentile": 10, "QueueCallTime_90th_percentile": 12, "QueueCallTime_95th_percentile": 13, "QueueCallTime_98th_percentile": 13, "QueueCallTime_99th_percentile": 13, "QueueCallTime_99.9th_percentile": 13, "QueueCallTime_TimeRangeCount_0-1": 130, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 130, "TotalCallTime_min": 0, "TotalCallTime_max": 26, "TotalCallTime_mean": 3, "TotalCallTime_25th_percentile": 6, "TotalCallTime_median": 13, "TotalCallTime_75th_percentile": 19, "TotalCallTime_90th_percentile": 23, "TotalCallTime_95th_percentile": 24, "TotalCallTime_98th_percentile": 25, "TotalCallTime_99th_percentile": 25, "TotalCallTime_99.9th_percentile": 25, "TotalCallTime_TimeRangeCount_0-1": 130, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 130, "ResponseSize_min": 0, "ResponseSize_max": 22858, "ResponseSize_mean": 4218, "ResponseSize_25th_percentile": 5714, "ResponseSize_median": 11429, "ResponseSize_75th_percentile": 17143, "ResponseSize_90th_percentile": 20572, "ResponseSize_95th_percentile": 21715, "ResponseSize_98th_percentile": 22400, "ResponseSize_99th_percentile": 22629, "ResponseSize_99.9th_percentile": 22835, "ResponseSize_SizeRangeCount_0-10": 130, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 130, "RequestSize_min": 81, "RequestSize_max": 8095, "RequestSize_mean": 281, "RequestSize_25th_percentile": 2084, "RequestSize_median": 4088, "RequestSize_75th_percentile": 6091, "RequestSize_90th_percentile": 7293, "RequestSize_95th_percentile": 7694, "RequestSize_98th_percentile": 7934, "RequestSize_99th_percentile": 8014, "RequestSize_99.9th_percentile": 8086, "RequestSize_SizeRangeCount_0-10": 130, "sentBytes": 424695 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "d1ba675678c9", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:57355", "tag.serverName": "d1ba675678c9,41485,1731042802970", "tag.clusterId": "22414151-4211-4121-a8ec-5820592eaa4e", "tag.Context": "regionserver", "tag.Hostname": "d1ba675678c9", "regionCount": 14, "storeCount": 13, "hlogFileCount": 2, "hlogFileSize": 0, "storeFileCount": 3, "maxStoreFileCount": 1, "memStoreSize": 0, "memStoreHeapSize": 3328, "memStoreOffHeapSize": 0, "storeFileSize": 45242, "storeFileSizeGrowthRate": 9048.4, "maxStoreFileAge": 500, "minStoreFileAge": 385, "avgStoreFileAge": 436, "numReferenceFiles": 0, "regionServerStartTime": 1731042802970, "averageRegionSize": 4524, "storeFileIndexSize": 1648, "staticIndexSize": 442, "staticBloomSize": 264, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 3332, "localBytesRead": 3332, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922038576, "blockCacheCount": 4, "blockCacheDataBlockCount": 4, "blockCacheSize": 708304, "blockCacheCountHitPercent": 95.23809523809523, "blockCacheExpressHitPercent": 95.23809523809523, "l1CacheSize": 708304, "l1CacheFreeSize": 922038576, "l1CacheCount": 4, "l1CacheEvictionCount": 0, "l1CacheHitCount": 80, "l1CacheMissCount": 4, "l1CacheHitRatio": 0.9523809523809523, "l1CacheMissRatio": 0.047619047619047616, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 1.8, "writeRequestRatePerSecond": 0.0, "ByteBuffAllocatorHeapAllocationBytes": 36711, "ByteBuffAllocatorPoolAllocationBytes": 1397760, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 1, "activeScanners": 0, "totalRequestCount": 38, "totalRowActionRequestCount": 9, "readRequestCount": 9, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 0, "rpcGetRequestCount": 0, "rpcFullScanRequestCount": 9, "rpcScanRequestCount": 30, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 8, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 80, "blockCacheHitCountPrimary": 80, "blockCacheHitCachingCount": 80, "blockCacheMissCount": 4, "blockCacheMissCountPrimary": 4, "blockCacheMissCachingCount": 4, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 2, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 40, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 264, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 28672, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 30, "ScanTime_min": 0, "ScanTime_max": 16, "ScanTime_mean": 2, "ScanTime_25th_percentile": 4, "ScanTime_median": 8, "ScanTime_75th_percentile": 12, "ScanTime_90th_percentile": 14, "ScanTime_95th_percentile": 15, "ScanTime_98th_percentile": 15, "ScanTime_99th_percentile": 15, "ScanTime_99.9th_percentile": 15, "ScanTime_TimeRangeCount_0-1": 30, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 8, "Put_min": 0, "Put_max": 2, "Put_mean": 1, "Put_25th_percentile": 0, "Put_median": 1, "Put_75th_percentile": 1, "Put_90th_percentile": 1, "Put_95th_percentile": 1, "Put_98th_percentile": 1, "Put_99th_percentile": 1, "Put_99.9th_percentile": 1, "Put_TimeRangeCount_0-1": 8, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 45242, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 28672, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 1, "FlushOutputSize_min": 45242, "FlushOutputSize_max": 45242, "FlushOutputSize_mean": 45242, "FlushOutputSize_25th_percentile": 45242, "FlushOutputSize_median": 45242, "FlushOutputSize_75th_percentile": 45242, "FlushOutputSize_90th_percentile": 45242, "FlushOutputSize_95th_percentile": 45242, "FlushOutputSize_98th_percentile": 45242, "FlushOutputSize_99th_percentile": 45242, "FlushOutputSize_99.9th_percentile": 45242, "FlushOutputSize_SizeRangeCount_0-10": 1, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 21, "ScanBlockBytesScanned_min": 28942, "ScanBlockBytesScanned_max": 28942, "ScanBlockBytesScanned_mean": 28942, "ScanBlockBytesScanned_25th_percentile": 28942, "ScanBlockBytesScanned_median": 28942, "ScanBlockBytesScanned_75th_percentile": 28942, "ScanBlockBytesScanned_90th_percentile": 28942, "ScanBlockBytesScanned_95th_percentile": 28942, "ScanBlockBytesScanned_98th_percentile": 28942, "ScanBlockBytesScanned_99th_percentile": 28942, "ScanBlockBytesScanned_99.9th_percentile": 28942, "ScanBlockBytesScanned_SizeRangeCount_0-10": 21, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 607782, "FlushMemstoreSize_num_ops": 1, "FlushMemstoreSize_min": 28672, "FlushMemstoreSize_max": 28672, "FlushMemstoreSize_mean": 28672, "FlushMemstoreSize_25th_percentile": 28672, "FlushMemstoreSize_median": 28672, "FlushMemstoreSize_75th_percentile": 28672, "FlushMemstoreSize_90th_percentile": 28672, "FlushMemstoreSize_95th_percentile": 28672, "FlushMemstoreSize_98th_percentile": 28672, "FlushMemstoreSize_99th_percentile": 28672, "FlushMemstoreSize_99.9th_percentile": 28672, "FlushMemstoreSize_SizeRangeCount_0-10": 1, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 0, "Get_min": 0, "Get_max": 0, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 1, "FlushTime_min": 244, "FlushTime_max": 244, "FlushTime_mean": 244, "FlushTime_25th_percentile": 244, "FlushTime_median": 244, "FlushTime_75th_percentile": 244, "FlushTime_90th_percentile": 244, "FlushTime_95th_percentile": 244, "FlushTime_98th_percentile": 244, "FlushTime_99th_percentile": 244, "FlushTime_99.9th_percentile": 244, "FlushTime_TimeRangeCount_0-1": 1, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 30, "ScanSize_min": 0, "ScanSize_max": 22731, "ScanSize_mean": 15911, "ScanSize_25th_percentile": 5682, "ScanSize_median": 11365, "ScanSize_75th_percentile": 17048, "ScanSize_90th_percentile": 20457, "ScanSize_95th_percentile": 21594, "ScanSize_98th_percentile": 22276, "ScanSize_99th_percentile": 22503, "ScanSize_99.9th_percentile": 22708, "ScanSize_SizeRangeCount_0-10": 30, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-11-08T05:13:30,942 WARN [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2470): Unable to report fatal error to master org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=d1ba675678c9:37587 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:344) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportRSFatalError(RegionServerStatusProtos.java:17290) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.abort(HRegionServer.java:2467) ~[classes/:?] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.abortRegionServer(SingleProcessHBaseCluster.java:192) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$2.run(SingleProcessHBaseCluster.java:185) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at java.security.AccessController.doPrivileged(AccessController.java:399) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:376) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.abort(SingleProcessHBaseCluster.java:182) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.handleException(AssignRegionHandler.java:173) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:106) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=d1ba675678c9:37587 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 15 more Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 15 more 2024-11-08T05:13:30,943 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] handler.AssignRegionHandler(132): Open test,jjj,1731042805591.a12ea80f78b339cfbfc0e18b5338df0c. 2024-11-08T05:13:30,943 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7752): Opening region: {ENCODED => a12ea80f78b339cfbfc0e18b5338df0c, NAME => 'test,jjj,1731042805591.a12ea80f78b339cfbfc0e18b5338df0c.', STARTKEY => 'jjj', ENDKEY => 'kkk'} 2024-11-08T05:13:30,944 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test a12ea80f78b339cfbfc0e18b5338df0c 2024-11-08T05:13:30,944 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(898): Instantiated test,jjj,1731042805591.a12ea80f78b339cfbfc0e18b5338df0c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T05:13:30,944 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7794): checking encryption for a12ea80f78b339cfbfc0e18b5338df0c 2024-11-08T05:13:30,944 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7797): checking classloading for a12ea80f78b339cfbfc0e18b5338df0c 2024-11-08T05:13:30,945 INFO [StoreOpener-a12ea80f78b339cfbfc0e18b5338df0c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region a12ea80f78b339cfbfc0e18b5338df0c 2024-11-08T05:13:30,947 INFO [StoreOpener-a12ea80f78b339cfbfc0e18b5338df0c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a12ea80f78b339cfbfc0e18b5338df0c columnFamilyName family 2024-11-08T05:13:30,947 DEBUG [StoreOpener-a12ea80f78b339cfbfc0e18b5338df0c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T05:13:30,948 INFO [StoreOpener-a12ea80f78b339cfbfc0e18b5338df0c-1 {}] regionserver.HStore(327): Store=a12ea80f78b339cfbfc0e18b5338df0c/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T05:13:30,948 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1038): replaying wal for a12ea80f78b339cfbfc0e18b5338df0c 2024-11-08T05:13:30,949 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/default/test/a12ea80f78b339cfbfc0e18b5338df0c 2024-11-08T05:13:30,950 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/default/test/a12ea80f78b339cfbfc0e18b5338df0c 2024-11-08T05:13:30,951 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1048): stopping wal replay for a12ea80f78b339cfbfc0e18b5338df0c 2024-11-08T05:13:30,951 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1060): Cleaning up temporary data for a12ea80f78b339cfbfc0e18b5338df0c 2024-11-08T05:13:30,953 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1093): writing seq id for a12ea80f78b339cfbfc0e18b5338df0c 2024-11-08T05:13:30,953 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1114): Opened a12ea80f78b339cfbfc0e18b5338df0c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65436009, jitterRate=-0.02492748200893402}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T05:13:30,954 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a12ea80f78b339cfbfc0e18b5338df0c 2024-11-08T05:13:30,954 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1006): Region open journal for a12ea80f78b339cfbfc0e18b5338df0c: Running coprocessor pre-open hook at 1731042810944Writing region info on filesystem at 1731042810944Initializing all the Stores at 1731042810945 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731042810945Cleaning up temporary data from old regions at 1731042810951 (+6 ms)Running coprocessor post-open hooks at 1731042810954 (+3 ms)Region opened successfully at 1731042810954 2024-11-08T05:13:30,955 WARN [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] handler.AssignRegionHandler(168): Fatal error occurred while opening region test,jjj,1731042805591.a12ea80f78b339cfbfc0e18b5338df0c., aborting... org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server d1ba675678c9,39291,1731042803020 aborting at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkOpen(RSRpcServices.java:1463) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2235) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T05:13:30,955 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionServer(2435): Abort already in progress. Ignoring the current request with reason: Failed to open region test,jjj,1731042805591.a12ea80f78b339cfbfc0e18b5338df0c. and can not recover 2024-11-08T05:13:30,955 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] handler.AssignRegionHandler(132): Open test,www,1731042805591.81806cb184c70c9dc0df408694092d85. 2024-11-08T05:13:30,955 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(7752): Opening region: {ENCODED => 81806cb184c70c9dc0df408694092d85, NAME => 'test,www,1731042805591.81806cb184c70c9dc0df408694092d85.', STARTKEY => 'www', ENDKEY => 'xxx'} 2024-11-08T05:13:30,956 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 81806cb184c70c9dc0df408694092d85 2024-11-08T05:13:30,956 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(898): Instantiated test,www,1731042805591.81806cb184c70c9dc0df408694092d85.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T05:13:30,956 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(7794): checking encryption for 81806cb184c70c9dc0df408694092d85 2024-11-08T05:13:30,956 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(7797): checking classloading for 81806cb184c70c9dc0df408694092d85 2024-11-08T05:13:30,957 INFO [StoreOpener-81806cb184c70c9dc0df408694092d85-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 81806cb184c70c9dc0df408694092d85 2024-11-08T05:13:30,959 INFO [StoreOpener-81806cb184c70c9dc0df408694092d85-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 81806cb184c70c9dc0df408694092d85 columnFamilyName family 2024-11-08T05:13:30,959 DEBUG [StoreOpener-81806cb184c70c9dc0df408694092d85-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T05:13:30,959 INFO [StoreOpener-81806cb184c70c9dc0df408694092d85-1 {}] regionserver.HStore(327): Store=81806cb184c70c9dc0df408694092d85/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T05:13:30,959 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1038): replaying wal for 81806cb184c70c9dc0df408694092d85 2024-11-08T05:13:30,960 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/default/test/81806cb184c70c9dc0df408694092d85 2024-11-08T05:13:30,962 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/default/test/81806cb184c70c9dc0df408694092d85 2024-11-08T05:13:30,962 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1048): stopping wal replay for 81806cb184c70c9dc0df408694092d85 2024-11-08T05:13:30,962 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1060): Cleaning up temporary data for 81806cb184c70c9dc0df408694092d85 2024-11-08T05:13:30,964 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1093): writing seq id for 81806cb184c70c9dc0df408694092d85 2024-11-08T05:13:30,965 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1114): Opened 81806cb184c70c9dc0df408694092d85; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68698963, jitterRate=0.023694321513175964}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T05:13:30,965 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 81806cb184c70c9dc0df408694092d85 2024-11-08T05:13:30,965 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1006): Region open journal for 81806cb184c70c9dc0df408694092d85: Running coprocessor pre-open hook at 1731042810956Writing region info on filesystem at 1731042810956Initializing all the Stores at 1731042810957 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731042810957Cleaning up temporary data from old regions at 1731042810962 (+5 ms)Running coprocessor post-open hooks at 1731042810965 (+3 ms)Region opened successfully at 1731042810965 2024-11-08T05:13:30,966 WARN [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] handler.AssignRegionHandler(168): Fatal error occurred while opening region test,www,1731042805591.81806cb184c70c9dc0df408694092d85., aborting... org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server d1ba675678c9,39291,1731042803020 aborting at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkOpen(RSRpcServices.java:1463) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2235) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T05:13:30,966 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegionServer(2435): Abort already in progress. Ignoring the current request with reason: Failed to open region test,www,1731042805591.81806cb184c70c9dc0df408694092d85. and can not recover 2024-11-08T05:13:30,966 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(132): Open test,qqq,1731042805591.7830dc6b1acea6e1d468f8008d8a82b9. 2024-11-08T05:13:30,966 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7752): Opening region: {ENCODED => 7830dc6b1acea6e1d468f8008d8a82b9, NAME => 'test,qqq,1731042805591.7830dc6b1acea6e1d468f8008d8a82b9.', STARTKEY => 'qqq', ENDKEY => 'rrr'} 2024-11-08T05:13:30,967 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 7830dc6b1acea6e1d468f8008d8a82b9 2024-11-08T05:13:30,967 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(898): Instantiated test,qqq,1731042805591.7830dc6b1acea6e1d468f8008d8a82b9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T05:13:30,967 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7794): checking encryption for 7830dc6b1acea6e1d468f8008d8a82b9 2024-11-08T05:13:30,967 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7797): checking classloading for 7830dc6b1acea6e1d468f8008d8a82b9 2024-11-08T05:13:30,968 INFO [StoreOpener-7830dc6b1acea6e1d468f8008d8a82b9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 7830dc6b1acea6e1d468f8008d8a82b9 2024-11-08T05:13:30,969 INFO [StoreOpener-7830dc6b1acea6e1d468f8008d8a82b9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7830dc6b1acea6e1d468f8008d8a82b9 columnFamilyName family 2024-11-08T05:13:30,969 DEBUG [StoreOpener-7830dc6b1acea6e1d468f8008d8a82b9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T05:13:30,970 INFO [StoreOpener-7830dc6b1acea6e1d468f8008d8a82b9-1 {}] regionserver.HStore(327): Store=7830dc6b1acea6e1d468f8008d8a82b9/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T05:13:30,970 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1038): replaying wal for 7830dc6b1acea6e1d468f8008d8a82b9 2024-11-08T05:13:30,971 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/default/test/7830dc6b1acea6e1d468f8008d8a82b9 2024-11-08T05:13:30,972 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/data/default/test/7830dc6b1acea6e1d468f8008d8a82b9 2024-11-08T05:13:30,973 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1048): stopping wal replay for 7830dc6b1acea6e1d468f8008d8a82b9 2024-11-08T05:13:30,973 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1060): Cleaning up temporary data for 7830dc6b1acea6e1d468f8008d8a82b9 2024-11-08T05:13:30,975 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1093): writing seq id for 7830dc6b1acea6e1d468f8008d8a82b9 2024-11-08T05:13:30,975 INFO [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1114): Opened 7830dc6b1acea6e1d468f8008d8a82b9; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73422894, jitterRate=0.09408637881278992}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T05:13:30,976 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7830dc6b1acea6e1d468f8008d8a82b9 2024-11-08T05:13:30,976 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1006): Region open journal for 7830dc6b1acea6e1d468f8008d8a82b9: Running coprocessor pre-open hook at 1731042810967Writing region info on filesystem at 1731042810967Initializing all the Stores at 1731042810968 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731042810968Cleaning up temporary data from old regions at 1731042810973 (+5 ms)Running coprocessor post-open hooks at 1731042810976 (+3 ms)Region opened successfully at 1731042810976 2024-11-08T05:13:30,977 WARN [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(168): Fatal error occurred while opening region test,qqq,1731042805591.7830dc6b1acea6e1d468f8008d8a82b9., aborting... org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server d1ba675678c9,39291,1731042803020 aborting at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkOpen(RSRpcServices.java:1463) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2235) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T05:13:30,977 DEBUG [RS_OPEN_REGION-regionserver/d1ba675678c9:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2435): Abort already in progress. Ignoring the current request with reason: Failed to open region test,qqq,1731042805591.7830dc6b1acea6e1d468f8008d8a82b9. and can not recover 2024-11-08T05:13:30,987 INFO [RS:2;d1ba675678c9:39291 {}] regionserver.HRegionServer(976): stopping server d1ba675678c9,39291,1731042803020; all regions closed. 2024-11-08T05:13:30,988 INFO [RS:1;d1ba675678c9:41485 {}] regionserver.HRegionServer(976): stopping server d1ba675678c9,41485,1731042802970; all regions closed. 2024-11-08T05:13:30,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741835_1011 (size=5988) 2024-11-08T05:13:30,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741835_1011 (size=5988) 2024-11-08T05:13:30,990 DEBUG [RS:2;d1ba675678c9:39291 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T05:13:30,990 INFO [RS:2;d1ba675678c9:39291 {}] regionserver.LeaseManager(133): Closed leases 2024-11-08T05:13:30,991 INFO [RS:2;d1ba675678c9:39291 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-08T05:13:30,991 INFO [RS:2;d1ba675678c9:39291 {}] hbase.ChoreService(370): Chore service for: regionserver/d1ba675678c9:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-08T05:13:30,991 INFO [RS:2;d1ba675678c9:39291 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-08T05:13:30,991 INFO [RS:2;d1ba675678c9:39291 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-08T05:13:30,991 INFO [RS:2;d1ba675678c9:39291 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-08T05:13:30,991 INFO [regionserver/d1ba675678c9:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-08T05:13:30,991 INFO [RS:2;d1ba675678c9:39291 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-08T05:13:30,992 INFO [RS:2;d1ba675678c9:39291 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:39291 2024-11-08T05:13:30,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741865_1041 (size=4201) 2024-11-08T05:13:30,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34565 is added to blk_1073741865_1041 (size=4201) 2024-11-08T05:13:30,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741865_1041 (size=4201) 2024-11-08T05:13:30,997 DEBUG [RS:1;d1ba675678c9:41485 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/oldWALs 2024-11-08T05:13:30,998 INFO [RS:1;d1ba675678c9:41485 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL d1ba675678c9%2C41485%2C1731042802970.meta:.meta(num 1731042808759) 2024-11-08T05:13:31,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741834_1010 (size=7074) 2024-11-08T05:13:31,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34565 is added to blk_1073741834_1010 (size=7074) 2024-11-08T05:13:31,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741834_1010 (size=7074) 2024-11-08T05:13:31,004 DEBUG [RS:1;d1ba675678c9:41485 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/oldWALs 2024-11-08T05:13:31,004 INFO [RS:1;d1ba675678c9:41485 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL d1ba675678c9%2C41485%2C1731042802970:(num 1731042804603) 2024-11-08T05:13:31,004 DEBUG [RS:1;d1ba675678c9:41485 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T05:13:31,004 INFO [RS:1;d1ba675678c9:41485 {}] regionserver.LeaseManager(133): Closed leases 2024-11-08T05:13:31,005 INFO [RS:1;d1ba675678c9:41485 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-08T05:13:31,005 INFO [RS:1;d1ba675678c9:41485 {}] hbase.ChoreService(370): Chore service for: regionserver/d1ba675678c9:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-08T05:13:31,005 INFO [RS:1;d1ba675678c9:41485 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-08T05:13:31,005 INFO [regionserver/d1ba675678c9:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-08T05:13:31,005 INFO [RS:1;d1ba675678c9:41485 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:41485 2024-11-08T05:13:31,007 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39291-0x10118dcefab0003, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/d1ba675678c9,39291,1731042803020 2024-11-08T05:13:31,007 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37587-0x10118dcefab0000, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-08T05:13:31,007 INFO [RS:2;d1ba675678c9:39291 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-08T05:13:31,018 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41485-0x10118dcefab0002, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/d1ba675678c9,41485,1731042802970 2024-11-08T05:13:31,018 INFO [RS:1;d1ba675678c9:41485 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-08T05:13:31,018 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [d1ba675678c9,41485,1731042802970] 2024-11-08T05:13:31,039 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/d1ba675678c9,41485,1731042802970 already deleted, retry=false 2024-11-08T05:13:31,039 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; d1ba675678c9,41485,1731042802970 expired; onlineServers=1 2024-11-08T05:13:31,040 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [d1ba675678c9,39291,1731042803020] 2024-11-08T05:13:31,050 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/d1ba675678c9,39291,1731042803020 already deleted, retry=false 2024-11-08T05:13:31,050 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; d1ba675678c9,39291,1731042803020 expired; onlineServers=0 2024-11-08T05:13:31,050 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'd1ba675678c9,37587,1731042801796' ***** 2024-11-08T05:13:31,050 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-08T05:13:31,050 INFO [M:0;d1ba675678c9:37587 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-08T05:13:31,051 INFO [M:0;d1ba675678c9:37587 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-08T05:13:31,051 DEBUG [M:0;d1ba675678c9:37587 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-08T05:13:31,051 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-08T05:13:31,051 DEBUG [M:0;d1ba675678c9:37587 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-08T05:13:31,051 DEBUG [master/d1ba675678c9:0:becomeActiveMaster-HFileCleaner.small.0-1731042804205 {}] cleaner.HFileCleaner(306): Exit Thread[master/d1ba675678c9:0:becomeActiveMaster-HFileCleaner.small.0-1731042804205,5,FailOnTimeoutGroup] 2024-11-08T05:13:31,051 DEBUG [master/d1ba675678c9:0:becomeActiveMaster-HFileCleaner.large.0-1731042804201 {}] cleaner.HFileCleaner(306): Exit Thread[master/d1ba675678c9:0:becomeActiveMaster-HFileCleaner.large.0-1731042804201,5,FailOnTimeoutGroup] 2024-11-08T05:13:31,052 INFO [M:0;d1ba675678c9:37587 {}] hbase.ChoreService(370): Chore service for: master/d1ba675678c9:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-08T05:13:31,052 INFO [M:0;d1ba675678c9:37587 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-08T05:13:31,052 DEBUG [M:0;d1ba675678c9:37587 {}] master.HMaster(1795): Stopping service threads 2024-11-08T05:13:31,052 INFO [M:0;d1ba675678c9:37587 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-08T05:13:31,052 INFO [M:0;d1ba675678c9:37587 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T05:13:31,053 INFO [M:0;d1ba675678c9:37587 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-08T05:13:31,053 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-08T05:13:31,060 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37587-0x10118dcefab0000, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-08T05:13:31,060 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37587-0x10118dcefab0000, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T05:13:31,060 DEBUG [M:0;d1ba675678c9:37587 {}] zookeeper.ZKUtil(347): master:37587-0x10118dcefab0000, quorum=127.0.0.1:57355, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-08T05:13:31,060 WARN [M:0;d1ba675678c9:37587 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-08T05:13:31,061 INFO [M:0;d1ba675678c9:37587 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/.lastflushedseqids 2024-11-08T05:13:31,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741870_1046 (size=1560) 2024-11-08T05:13:31,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34565 is added to blk_1073741870_1046 (size=1560) 2024-11-08T05:13:31,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741870_1046 (size=1560) 2024-11-08T05:13:31,076 INFO [M:0;d1ba675678c9:37587 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-08T05:13:31,076 INFO [M:0;d1ba675678c9:37587 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-08T05:13:31,076 DEBUG [M:0;d1ba675678c9:37587 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-08T05:13:31,076 INFO [M:0;d1ba675678c9:37587 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T05:13:31,076 DEBUG [M:0;d1ba675678c9:37587 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T05:13:31,076 DEBUG [M:0;d1ba675678c9:37587 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-08T05:13:31,076 DEBUG [M:0;d1ba675678c9:37587 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T05:13:31,076 INFO [M:0;d1ba675678c9:37587 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=248.71 KB heapSize=302.27 KB 2024-11-08T05:13:31,095 DEBUG [M:0;d1ba675678c9:37587 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/3970b40c735342f7b32712747b0bc0e5 is 82, key is hbase:meta,,1/info:regioninfo/1731042809079/Put/seqid=0 2024-11-08T05:13:31,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34565 is added to blk_1073741871_1047 (size=6071) 2024-11-08T05:13:31,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741871_1047 (size=6071) 2024-11-08T05:13:31,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741871_1047 (size=6071) 2024-11-08T05:13:31,102 INFO [M:0;d1ba675678c9:37587 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=571 (bloomFilter=true), to=hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/3970b40c735342f7b32712747b0bc0e5 2024-11-08T05:13:31,126 DEBUG [M:0;d1ba675678c9:37587 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/cc00de2a1b904ddc901d0d572708134a is 2987, key is \x00\x00\x00\x00\x00\x00\x009/proc:d/1731042810748/Put/seqid=0 2024-11-08T05:13:31,129 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39291-0x10118dcefab0003, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T05:13:31,129 INFO [RS:2;d1ba675678c9:39291 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-08T05:13:31,129 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39291-0x10118dcefab0003, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T05:13:31,129 INFO [RS:2;d1ba675678c9:39291 {}] regionserver.HRegionServer(1031): Exiting; stopping=d1ba675678c9,39291,1731042803020; zookeeper connection closed. 2024-11-08T05:13:31,129 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@4b147593 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@4b147593 2024-11-08T05:13:31,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741872_1048 (size=12797) 2024-11-08T05:13:31,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34565 is added to blk_1073741872_1048 (size=12797) 2024-11-08T05:13:31,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741872_1048 (size=12797) 2024-11-08T05:13:31,133 INFO [M:0;d1ba675678c9:37587 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=247.44 KB at sequenceid=571 (bloomFilter=true), to=hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/cc00de2a1b904ddc901d0d572708134a 2024-11-08T05:13:31,139 INFO [RS:1;d1ba675678c9:41485 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-08T05:13:31,139 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41485-0x10118dcefab0002, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T05:13:31,139 INFO [RS:1;d1ba675678c9:41485 {}] regionserver.HRegionServer(1031): Exiting; stopping=d1ba675678c9,41485,1731042802970; zookeeper connection closed. 2024-11-08T05:13:31,139 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41485-0x10118dcefab0002, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T05:13:31,140 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@b59e23a {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@b59e23a 2024-11-08T05:13:31,140 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-08T05:13:31,155 DEBUG [M:0;d1ba675678c9:37587 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/1327b91000ce4451967739986b3809c5 is 69, key is d1ba675678c9,39291,1731042803020/rs:state/1731042804317/Put/seqid=0 2024-11-08T05:13:31,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34565 is added to blk_1073741873_1049 (size=5442) 2024-11-08T05:13:31,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741873_1049 (size=5442) 2024-11-08T05:13:31,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741873_1049 (size=5442) 2024-11-08T05:13:31,163 INFO [M:0;d1ba675678c9:37587 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=571 (bloomFilter=true), to=hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/1327b91000ce4451967739986b3809c5 2024-11-08T05:13:31,170 INFO [M:0;d1ba675678c9:37587 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 1327b91000ce4451967739986b3809c5 2024-11-08T05:13:31,185 DEBUG [M:0;d1ba675678c9:37587 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/067bec9718a6471690d29548c22db5e2 is 52, key is load_balancer_on/state:d/1731042807945/Put/seqid=0 2024-11-08T05:13:31,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741874_1050 (size=5058) 2024-11-08T05:13:31,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34565 is added to blk_1073741874_1050 (size=5058) 2024-11-08T05:13:31,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741874_1050 (size=5058) 2024-11-08T05:13:31,195 INFO [M:0;d1ba675678c9:37587 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=571 (bloomFilter=true), to=hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/067bec9718a6471690d29548c22db5e2 2024-11-08T05:13:31,202 DEBUG [M:0;d1ba675678c9:37587 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/3970b40c735342f7b32712747b0bc0e5 as hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/3970b40c735342f7b32712747b0bc0e5 2024-11-08T05:13:31,209 INFO [M:0;d1ba675678c9:37587 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/3970b40c735342f7b32712747b0bc0e5, entries=14, sequenceid=571, filesize=5.9 K 2024-11-08T05:13:31,210 DEBUG [M:0;d1ba675678c9:37587 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/cc00de2a1b904ddc901d0d572708134a as hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/cc00de2a1b904ddc901d0d572708134a 2024-11-08T05:13:31,217 INFO [M:0;d1ba675678c9:37587 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/cc00de2a1b904ddc901d0d572708134a, entries=79, sequenceid=571, filesize=12.5 K 2024-11-08T05:13:31,218 DEBUG [M:0;d1ba675678c9:37587 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/1327b91000ce4451967739986b3809c5 as hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/1327b91000ce4451967739986b3809c5 2024-11-08T05:13:31,225 INFO [M:0;d1ba675678c9:37587 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 1327b91000ce4451967739986b3809c5 2024-11-08T05:13:31,225 INFO [M:0;d1ba675678c9:37587 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/1327b91000ce4451967739986b3809c5, entries=3, sequenceid=571, filesize=5.3 K 2024-11-08T05:13:31,227 DEBUG [M:0;d1ba675678c9:37587 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/067bec9718a6471690d29548c22db5e2 as hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/067bec9718a6471690d29548c22db5e2 2024-11-08T05:13:31,233 INFO [M:0;d1ba675678c9:37587 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45429/user/jenkins/test-data/30a8b6f0-5f7e-b803-6643-5fa6e1fd3e5c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/067bec9718a6471690d29548c22db5e2, entries=1, sequenceid=571, filesize=4.9 K 2024-11-08T05:13:31,234 INFO [M:0;d1ba675678c9:37587 {}] regionserver.HRegion(3140): Finished flush of dataSize ~248.71 KB/254679, heapSize ~302.20 KB/309456, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 158ms, sequenceid=571, compaction requested=false 2024-11-08T05:13:31,236 INFO [M:0;d1ba675678c9:37587 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T05:13:31,236 DEBUG [M:0;d1ba675678c9:37587 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731042811076Disabling compacts and flushes for region at 1731042811076Disabling writes for close at 1731042811076Obtaining lock to block concurrent updates at 1731042811076Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731042811076Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=254679, getHeapSize=309456, getOffHeapSize=0, getCellsCount=710 at 1731042811077 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731042811077Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731042811078 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731042811095 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731042811095Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731042811108 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731042811125 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731042811125Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731042811139 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731042811154 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731042811154Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1731042811170 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1731042811184 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1731042811184Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3409dbc6: reopening flushed file at 1731042811201 (+17 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6c13bf0d: reopening flushed file at 1731042811209 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@60fec0c2: reopening flushed file at 1731042811217 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7146488d: reopening flushed file at 1731042811226 (+9 ms)Finished flush of dataSize ~248.71 KB/254679, heapSize ~302.20 KB/309456, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 158ms, sequenceid=571, compaction requested=false at 1731042811235 (+9 ms)Writing region close event to WAL at 1731042811236 (+1 ms)Closed at 1731042811236 2024-11-08T05:13:31,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741830_1006 (size=292803) 2024-11-08T05:13:31,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741830_1006 (size=292803) 2024-11-08T05:13:31,240 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-08T05:13:31,240 INFO [M:0;d1ba675678c9:37587 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-08T05:13:31,241 INFO [M:0;d1ba675678c9:37587 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:37587 2024-11-08T05:13:31,241 INFO [M:0;d1ba675678c9:37587 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-08T05:13:31,350 INFO [M:0;d1ba675678c9:37587 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-08T05:13:31,350 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37587-0x10118dcefab0000, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T05:13:31,350 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37587-0x10118dcefab0000, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T05:13:31,357 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@67ddfd25{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-08T05:13:31,360 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@30e38263{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-08T05:13:31,360 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-08T05:13:31,360 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@52cf89e6{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-08T05:13:31,360 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@11ebaa12{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/494744ca-67d2-5496-2c52-6b1334939ddf/hadoop.log.dir/,STOPPED} 2024-11-08T05:13:31,363 WARN [BP-148118038-172.17.0.3-1731042796219 heartbeating to localhost/127.0.0.1:45429 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-08T05:13:31,363 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-08T05:13:31,363 WARN [BP-148118038-172.17.0.3-1731042796219 heartbeating to localhost/127.0.0.1:45429 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-148118038-172.17.0.3-1731042796219 (Datanode Uuid a7d9b40c-b323-4028-a313-57a6ed1b1f07) service to localhost/127.0.0.1:45429 2024-11-08T05:13:31,363 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-08T05:13:31,365 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/494744ca-67d2-5496-2c52-6b1334939ddf/cluster_b5b9bc45-3113-1e92-d4ad-d82b8cd072e6/data/data5/current/BP-148118038-172.17.0.3-1731042796219 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-08T05:13:31,365 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/494744ca-67d2-5496-2c52-6b1334939ddf/cluster_b5b9bc45-3113-1e92-d4ad-d82b8cd072e6/data/data6/current/BP-148118038-172.17.0.3-1731042796219 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-08T05:13:31,365 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-08T05:13:31,368 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7cfccf27{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-08T05:13:31,368 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6c41bbb7{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-08T05:13:31,368 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-08T05:13:31,368 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@12f16529{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-08T05:13:31,369 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@34a3e1ff{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/494744ca-67d2-5496-2c52-6b1334939ddf/hadoop.log.dir/,STOPPED} 2024-11-08T05:13:31,370 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-08T05:13:31,370 WARN [BP-148118038-172.17.0.3-1731042796219 heartbeating to localhost/127.0.0.1:45429 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-08T05:13:31,370 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-08T05:13:31,370 WARN [BP-148118038-172.17.0.3-1731042796219 heartbeating to localhost/127.0.0.1:45429 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-148118038-172.17.0.3-1731042796219 (Datanode Uuid 7daa6c49-f147-4f86-84ea-b92d941c350c) service to localhost/127.0.0.1:45429 2024-11-08T05:13:31,371 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/494744ca-67d2-5496-2c52-6b1334939ddf/cluster_b5b9bc45-3113-1e92-d4ad-d82b8cd072e6/data/data3/current/BP-148118038-172.17.0.3-1731042796219 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-08T05:13:31,371 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/494744ca-67d2-5496-2c52-6b1334939ddf/cluster_b5b9bc45-3113-1e92-d4ad-d82b8cd072e6/data/data4/current/BP-148118038-172.17.0.3-1731042796219 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-08T05:13:31,371 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-08T05:13:31,373 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2226d6a6{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-08T05:13:31,374 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@1da1f773{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-08T05:13:31,374 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-08T05:13:31,374 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1c5d2af1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-08T05:13:31,374 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@739a104e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/494744ca-67d2-5496-2c52-6b1334939ddf/hadoop.log.dir/,STOPPED} 2024-11-08T05:13:31,375 WARN [BP-148118038-172.17.0.3-1731042796219 heartbeating to localhost/127.0.0.1:45429 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-08T05:13:31,375 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-08T05:13:31,375 WARN [BP-148118038-172.17.0.3-1731042796219 heartbeating to localhost/127.0.0.1:45429 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-148118038-172.17.0.3-1731042796219 (Datanode Uuid 4cfcc222-3171-4d68-9431-a0e80dfd6ba6) service to localhost/127.0.0.1:45429 2024-11-08T05:13:31,375 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-08T05:13:31,376 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/494744ca-67d2-5496-2c52-6b1334939ddf/cluster_b5b9bc45-3113-1e92-d4ad-d82b8cd072e6/data/data1/current/BP-148118038-172.17.0.3-1731042796219 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-08T05:13:31,376 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/494744ca-67d2-5496-2c52-6b1334939ddf/cluster_b5b9bc45-3113-1e92-d4ad-d82b8cd072e6/data/data2/current/BP-148118038-172.17.0.3-1731042796219 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-08T05:13:31,376 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-08T05:13:31,384 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@376fa8f3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-08T05:13:31,384 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@672b88ea{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-08T05:13:31,385 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-08T05:13:31,385 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@76beb053{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-08T05:13:31,385 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@265b24f1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/494744ca-67d2-5496-2c52-6b1334939ddf/hadoop.log.dir/,STOPPED} 2024-11-08T05:13:31,395 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-08T05:13:31,442 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down