2024-12-07 01:14:15,046 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 2024-12-07 01:14:15,059 main DEBUG Took 0.011181 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-07 01:14:15,060 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-07 01:14:15,060 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-07 01:14:15,061 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-07 01:14:15,062 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 01:14:15,068 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-07 01:14:15,079 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 01:14:15,081 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 01:14:15,081 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 01:14:15,082 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 01:14:15,082 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 01:14:15,083 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 01:14:15,084 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 01:14:15,084 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 01:14:15,085 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 01:14:15,085 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 01:14:15,086 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 01:14:15,086 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 01:14:15,087 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 01:14:15,087 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 01:14:15,088 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 01:14:15,088 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 01:14:15,089 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 01:14:15,089 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 01:14:15,090 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 01:14:15,090 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 01:14:15,090 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 01:14:15,091 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 01:14:15,091 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 01:14:15,092 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 01:14:15,092 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 01:14:15,093 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-07 01:14:15,095 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 01:14:15,096 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-07 01:14:15,098 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-07 01:14:15,099 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-07 01:14:15,100 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-07 01:14:15,101 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-07 01:14:15,109 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-07 01:14:15,112 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-07 01:14:15,113 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-07 01:14:15,114 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-07 01:14:15,114 main DEBUG createAppenders(={Console}) 2024-12-07 01:14:15,115 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 initialized 2024-12-07 01:14:15,116 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 2024-12-07 01:14:15,116 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 OK. 2024-12-07 01:14:15,117 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-07 01:14:15,117 main DEBUG OutputStream closed 2024-12-07 01:14:15,118 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-07 01:14:15,118 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-07 01:14:15,119 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@c88a337 OK 2024-12-07 01:14:15,202 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-07 01:14:15,203 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-07 01:14:15,205 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-07 01:14:15,206 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-07 01:14:15,207 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-07 01:14:15,207 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-07 01:14:15,208 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-07 01:14:15,208 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-07 01:14:15,208 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-07 01:14:15,209 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-07 01:14:15,209 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-07 01:14:15,210 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-07 01:14:15,210 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-07 01:14:15,211 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-07 01:14:15,211 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-07 01:14:15,211 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-07 01:14:15,212 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-07 01:14:15,212 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-07 01:14:15,214 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-07 01:14:15,215 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@13330ac6) with optional ClassLoader: null 2024-12-07 01:14:15,215 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-07 01:14:15,216 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@13330ac6] started OK. 2024-12-07T01:14:15,229 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.master.assignment.TestRollbackSCP timeout: 13 mins 2024-12-07 01:14:15,231 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-07 01:14:15,232 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-07T01:14:15,464 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4e4c318f-1ef6-1cff-e733-20a2b141900e 2024-12-07T01:14:15,485 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=class org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$HMasterForTest, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-07T01:14:15,505 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4e4c318f-1ef6-1cff-e733-20a2b141900e/cluster_25b350fe-02a5-c1aa-017d-c8a3f951e395, deleteOnExit=true 2024-12-07T01:14:15,505 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-07T01:14:15,507 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4e4c318f-1ef6-1cff-e733-20a2b141900e/test.cache.data in system properties and HBase conf 2024-12-07T01:14:15,507 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4e4c318f-1ef6-1cff-e733-20a2b141900e/hadoop.tmp.dir in system properties and HBase conf 2024-12-07T01:14:15,508 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4e4c318f-1ef6-1cff-e733-20a2b141900e/hadoop.log.dir in system properties and HBase conf 2024-12-07T01:14:15,509 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4e4c318f-1ef6-1cff-e733-20a2b141900e/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-07T01:14:15,510 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4e4c318f-1ef6-1cff-e733-20a2b141900e/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-07T01:14:15,510 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-07T01:14:15,618 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-07T01:14:15,764 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-07T01:14:15,768 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4e4c318f-1ef6-1cff-e733-20a2b141900e/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-07T01:14:15,769 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4e4c318f-1ef6-1cff-e733-20a2b141900e/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-07T01:14:15,770 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4e4c318f-1ef6-1cff-e733-20a2b141900e/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-07T01:14:15,770 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4e4c318f-1ef6-1cff-e733-20a2b141900e/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-07T01:14:15,771 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4e4c318f-1ef6-1cff-e733-20a2b141900e/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-07T01:14:15,771 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4e4c318f-1ef6-1cff-e733-20a2b141900e/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-07T01:14:15,772 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4e4c318f-1ef6-1cff-e733-20a2b141900e/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-07T01:14:15,772 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4e4c318f-1ef6-1cff-e733-20a2b141900e/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-07T01:14:15,773 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4e4c318f-1ef6-1cff-e733-20a2b141900e/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-07T01:14:15,773 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4e4c318f-1ef6-1cff-e733-20a2b141900e/nfs.dump.dir in system properties and HBase conf 2024-12-07T01:14:15,774 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4e4c318f-1ef6-1cff-e733-20a2b141900e/java.io.tmpdir in system properties and HBase conf 2024-12-07T01:14:15,774 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4e4c318f-1ef6-1cff-e733-20a2b141900e/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-07T01:14:15,775 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4e4c318f-1ef6-1cff-e733-20a2b141900e/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-07T01:14:15,775 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4e4c318f-1ef6-1cff-e733-20a2b141900e/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-07T01:14:16,865 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-07T01:14:16,935 INFO [Time-limited test {}] log.Log(170): Logging initialized @2615ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-07T01:14:17,009 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-07T01:14:17,073 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-07T01:14:17,107 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-07T01:14:17,108 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-07T01:14:17,111 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-07T01:14:17,127 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-07T01:14:17,130 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3912242{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4e4c318f-1ef6-1cff-e733-20a2b141900e/hadoop.log.dir/,AVAILABLE} 2024-12-07T01:14:17,132 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@595320c8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-07T01:14:17,334 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@64418798{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4e4c318f-1ef6-1cff-e733-20a2b141900e/java.io.tmpdir/jetty-localhost-45603-hadoop-hdfs-3_4_1-tests_jar-_-any-7594617789103117857/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-07T01:14:17,342 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@11744524{HTTP/1.1, (http/1.1)}{localhost:45603} 2024-12-07T01:14:17,343 INFO [Time-limited test {}] server.Server(415): Started @3023ms 2024-12-07T01:14:17,984 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-07T01:14:17,993 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-07T01:14:17,995 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-07T01:14:17,995 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-07T01:14:17,995 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-07T01:14:17,996 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3a300759{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4e4c318f-1ef6-1cff-e733-20a2b141900e/hadoop.log.dir/,AVAILABLE} 2024-12-07T01:14:17,997 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@474d518f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-07T01:14:18,126 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@79b30fb0{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4e4c318f-1ef6-1cff-e733-20a2b141900e/java.io.tmpdir/jetty-localhost-41849-hadoop-hdfs-3_4_1-tests_jar-_-any-11775325870193364786/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-07T01:14:18,127 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@548cd725{HTTP/1.1, (http/1.1)}{localhost:41849} 2024-12-07T01:14:18,127 INFO [Time-limited test {}] server.Server(415): Started @3808ms 2024-12-07T01:14:18,171 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-07T01:14:18,273 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-07T01:14:18,278 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-07T01:14:18,280 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-07T01:14:18,280 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-07T01:14:18,281 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-07T01:14:18,282 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@29652cf4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4e4c318f-1ef6-1cff-e733-20a2b141900e/hadoop.log.dir/,AVAILABLE} 2024-12-07T01:14:18,282 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1b0a32ba{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-07T01:14:18,380 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@34fcb585{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4e4c318f-1ef6-1cff-e733-20a2b141900e/java.io.tmpdir/jetty-localhost-36393-hadoop-hdfs-3_4_1-tests_jar-_-any-10623869562502901439/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-07T01:14:18,380 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@389bb6f7{HTTP/1.1, (http/1.1)}{localhost:36393} 2024-12-07T01:14:18,381 INFO [Time-limited test {}] server.Server(415): Started @4061ms 2024-12-07T01:14:18,384 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-07T01:14:18,421 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-07T01:14:18,427 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-07T01:14:18,428 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-07T01:14:18,429 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-07T01:14:18,429 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-07T01:14:18,431 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@290e2eec{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4e4c318f-1ef6-1cff-e733-20a2b141900e/hadoop.log.dir/,AVAILABLE} 2024-12-07T01:14:18,432 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@417efacb{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-07T01:14:18,530 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3c1f701{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4e4c318f-1ef6-1cff-e733-20a2b141900e/java.io.tmpdir/jetty-localhost-38973-hadoop-hdfs-3_4_1-tests_jar-_-any-11249607426209625330/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-07T01:14:18,530 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3d599bb4{HTTP/1.1, (http/1.1)}{localhost:38973} 2024-12-07T01:14:18,531 INFO [Time-limited test {}] server.Server(415): Started @4211ms 2024-12-07T01:14:18,533 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-07T01:14:19,787 WARN [Thread-125 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4e4c318f-1ef6-1cff-e733-20a2b141900e/cluster_25b350fe-02a5-c1aa-017d-c8a3f951e395/data/data2/current/BP-1976987596-172.17.0.3-1733534056320/current, will proceed with Du for space computation calculation, 2024-12-07T01:14:19,787 WARN [Thread-122 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4e4c318f-1ef6-1cff-e733-20a2b141900e/cluster_25b350fe-02a5-c1aa-017d-c8a3f951e395/data/data3/current/BP-1976987596-172.17.0.3-1733534056320/current, will proceed with Du for space computation calculation, 2024-12-07T01:14:19,787 WARN [Thread-123 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4e4c318f-1ef6-1cff-e733-20a2b141900e/cluster_25b350fe-02a5-c1aa-017d-c8a3f951e395/data/data4/current/BP-1976987596-172.17.0.3-1733534056320/current, will proceed with Du for space computation calculation, 2024-12-07T01:14:19,787 WARN [Thread-124 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4e4c318f-1ef6-1cff-e733-20a2b141900e/cluster_25b350fe-02a5-c1aa-017d-c8a3f951e395/data/data1/current/BP-1976987596-172.17.0.3-1733534056320/current, will proceed with Du for space computation calculation, 2024-12-07T01:14:19,825 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-07T01:14:19,825 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-07T01:14:19,865 WARN [Thread-142 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4e4c318f-1ef6-1cff-e733-20a2b141900e/cluster_25b350fe-02a5-c1aa-017d-c8a3f951e395/data/data5/current/BP-1976987596-172.17.0.3-1733534056320/current, will proceed with Du for space computation calculation, 2024-12-07T01:14:19,865 WARN [Thread-143 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4e4c318f-1ef6-1cff-e733-20a2b141900e/cluster_25b350fe-02a5-c1aa-017d-c8a3f951e395/data/data6/current/BP-1976987596-172.17.0.3-1733534056320/current, will proceed with Du for space computation calculation, 2024-12-07T01:14:19,874 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xfa9954c10864c366 with lease ID 0x38628dbbbafe62e5: Processing first storage report for DS-b9cb305a-a2e1-439c-816b-c45d44c27920 from datanode DatanodeRegistration(127.0.0.1:37235, datanodeUuid=8b81b8a3-bf4e-48e3-8ddb-69e75519470d, infoPort=39885, infoSecurePort=0, ipcPort=44601, storageInfo=lv=-57;cid=testClusterID;nsid=446654696;c=1733534056320) 2024-12-07T01:14:19,875 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xfa9954c10864c366 with lease ID 0x38628dbbbafe62e5: from storage DS-b9cb305a-a2e1-439c-816b-c45d44c27920 node DatanodeRegistration(127.0.0.1:37235, datanodeUuid=8b81b8a3-bf4e-48e3-8ddb-69e75519470d, infoPort=39885, infoSecurePort=0, ipcPort=44601, storageInfo=lv=-57;cid=testClusterID;nsid=446654696;c=1733534056320), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-07T01:14:19,876 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x765aefb16a569d20 with lease ID 0x38628dbbbafe62e4: Processing first storage report for DS-59048819-6295-483c-8edd-efcadeb66bf1 from datanode DatanodeRegistration(127.0.0.1:40127, datanodeUuid=48fe2fef-5e13-4e48-b03f-fb4a2b850877, infoPort=45341, infoSecurePort=0, ipcPort=45201, storageInfo=lv=-57;cid=testClusterID;nsid=446654696;c=1733534056320) 2024-12-07T01:14:19,876 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x765aefb16a569d20 with lease ID 0x38628dbbbafe62e4: from storage DS-59048819-6295-483c-8edd-efcadeb66bf1 node DatanodeRegistration(127.0.0.1:40127, datanodeUuid=48fe2fef-5e13-4e48-b03f-fb4a2b850877, infoPort=45341, infoSecurePort=0, ipcPort=45201, storageInfo=lv=-57;cid=testClusterID;nsid=446654696;c=1733534056320), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-07T01:14:19,876 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xfa9954c10864c366 with lease ID 0x38628dbbbafe62e5: Processing first storage report for DS-b76c20ef-dde7-4324-817e-90fb6188120f from datanode DatanodeRegistration(127.0.0.1:37235, datanodeUuid=8b81b8a3-bf4e-48e3-8ddb-69e75519470d, infoPort=39885, infoSecurePort=0, ipcPort=44601, storageInfo=lv=-57;cid=testClusterID;nsid=446654696;c=1733534056320) 2024-12-07T01:14:19,876 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xfa9954c10864c366 with lease ID 0x38628dbbbafe62e5: from storage DS-b76c20ef-dde7-4324-817e-90fb6188120f node DatanodeRegistration(127.0.0.1:37235, datanodeUuid=8b81b8a3-bf4e-48e3-8ddb-69e75519470d, infoPort=39885, infoSecurePort=0, ipcPort=44601, storageInfo=lv=-57;cid=testClusterID;nsid=446654696;c=1733534056320), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-07T01:14:19,876 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x765aefb16a569d20 with lease ID 0x38628dbbbafe62e4: Processing first storage report for DS-db47333b-c9d2-4e34-bf28-ccfbb356c2d6 from datanode DatanodeRegistration(127.0.0.1:40127, datanodeUuid=48fe2fef-5e13-4e48-b03f-fb4a2b850877, infoPort=45341, infoSecurePort=0, ipcPort=45201, storageInfo=lv=-57;cid=testClusterID;nsid=446654696;c=1733534056320) 2024-12-07T01:14:19,877 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x765aefb16a569d20 with lease ID 0x38628dbbbafe62e4: from storage DS-db47333b-c9d2-4e34-bf28-ccfbb356c2d6 node DatanodeRegistration(127.0.0.1:40127, datanodeUuid=48fe2fef-5e13-4e48-b03f-fb4a2b850877, infoPort=45341, infoSecurePort=0, ipcPort=45201, storageInfo=lv=-57;cid=testClusterID;nsid=446654696;c=1733534056320), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-07T01:14:19,886 WARN [Thread-103 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-07T01:14:19,892 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x6d3a0943cbd28ef1 with lease ID 0x38628dbbbafe62e6: Processing first storage report for DS-f5ad7f4d-d62b-424d-bad1-dbf9e5694bea from datanode DatanodeRegistration(127.0.0.1:43991, datanodeUuid=41e996fa-a8fa-4e8a-9869-a72e6d9121e5, infoPort=40865, infoSecurePort=0, ipcPort=37781, storageInfo=lv=-57;cid=testClusterID;nsid=446654696;c=1733534056320) 2024-12-07T01:14:19,892 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x6d3a0943cbd28ef1 with lease ID 0x38628dbbbafe62e6: from storage DS-f5ad7f4d-d62b-424d-bad1-dbf9e5694bea node DatanodeRegistration(127.0.0.1:43991, datanodeUuid=41e996fa-a8fa-4e8a-9869-a72e6d9121e5, infoPort=40865, infoSecurePort=0, ipcPort=37781, storageInfo=lv=-57;cid=testClusterID;nsid=446654696;c=1733534056320), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-07T01:14:19,892 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x6d3a0943cbd28ef1 with lease ID 0x38628dbbbafe62e6: Processing first storage report for DS-723625e2-95fc-40f8-b85a-025f7f81814a from datanode DatanodeRegistration(127.0.0.1:43991, datanodeUuid=41e996fa-a8fa-4e8a-9869-a72e6d9121e5, infoPort=40865, infoSecurePort=0, ipcPort=37781, storageInfo=lv=-57;cid=testClusterID;nsid=446654696;c=1733534056320) 2024-12-07T01:14:19,893 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x6d3a0943cbd28ef1 with lease ID 0x38628dbbbafe62e6: from storage DS-723625e2-95fc-40f8-b85a-025f7f81814a node DatanodeRegistration(127.0.0.1:43991, datanodeUuid=41e996fa-a8fa-4e8a-9869-a72e6d9121e5, infoPort=40865, infoSecurePort=0, ipcPort=37781, storageInfo=lv=-57;cid=testClusterID;nsid=446654696;c=1733534056320), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-07T01:14:19,961 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4e4c318f-1ef6-1cff-e733-20a2b141900e 2024-12-07T01:14:20,035 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4e4c318f-1ef6-1cff-e733-20a2b141900e/cluster_25b350fe-02a5-c1aa-017d-c8a3f951e395/zookeeper_0, clientPort=61706, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4e4c318f-1ef6-1cff-e733-20a2b141900e/cluster_25b350fe-02a5-c1aa-017d-c8a3f951e395/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4e4c318f-1ef6-1cff-e733-20a2b141900e/cluster_25b350fe-02a5-c1aa-017d-c8a3f951e395/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-07T01:14:20,044 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=61706 2024-12-07T01:14:20,058 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-07T01:14:20,061 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-07T01:14:20,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37235 is added to blk_1073741825_1001 (size=7) 2024-12-07T01:14:20,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741825_1001 (size=7) 2024-12-07T01:14:20,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43991 is added to blk_1073741825_1001 (size=7) 2024-12-07T01:14:20,703 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323 with version=8 2024-12-07T01:14:20,703 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/hbase-staging 2024-12-07T01:14:20,771 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-07T01:14:21,005 INFO [Time-limited test {}] client.ConnectionUtils(128): master/ec1863dc21e5:0 server-side Connection retries=45 2024-12-07T01:14:21,014 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-07T01:14:21,015 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-07T01:14:21,019 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-07T01:14:21,019 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-07T01:14:21,019 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-07T01:14:21,127 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-07T01:14:21,182 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-07T01:14:21,189 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-07T01:14:21,192 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-07T01:14:21,212 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 2981 (auto-detected) 2024-12-07T01:14:21,213 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-12-07T01:14:21,228 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:46141 2024-12-07T01:14:21,246 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:46141 connecting to ZooKeeper ensemble=127.0.0.1:61706 2024-12-07T01:14:21,369 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:461410x0, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-07T01:14:21,373 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:46141-0x101ad5a899b0000 connected 2024-12-07T01:14:21,469 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-07T01:14:21,473 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-07T01:14:21,485 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:46141-0x101ad5a899b0000, quorum=127.0.0.1:61706, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-07T01:14:21,489 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323, hbase.cluster.distributed=false 2024-12-07T01:14:21,510 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:46141-0x101ad5a899b0000, quorum=127.0.0.1:61706, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-07T01:14:21,514 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46141 2024-12-07T01:14:21,515 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46141 2024-12-07T01:14:21,516 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46141 2024-12-07T01:14:21,516 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46141 2024-12-07T01:14:21,517 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46141 2024-12-07T01:14:21,615 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/ec1863dc21e5:0 server-side Connection retries=45 2024-12-07T01:14:21,616 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-07T01:14:21,617 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-07T01:14:21,617 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-07T01:14:21,617 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-07T01:14:21,617 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-07T01:14:21,620 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-07T01:14:21,623 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-07T01:14:21,624 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:42915 2024-12-07T01:14:21,625 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:42915 connecting to ZooKeeper ensemble=127.0.0.1:61706 2024-12-07T01:14:21,626 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-07T01:14:21,629 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-07T01:14:21,665 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:429150x0, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-07T01:14:21,666 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:429150x0, quorum=127.0.0.1:61706, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-07T01:14:21,667 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:42915-0x101ad5a899b0001 connected 2024-12-07T01:14:21,669 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-07T01:14:21,676 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-07T01:14:21,678 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42915-0x101ad5a899b0001, quorum=127.0.0.1:61706, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-07T01:14:21,684 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42915-0x101ad5a899b0001, quorum=127.0.0.1:61706, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-07T01:14:21,685 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42915 2024-12-07T01:14:21,685 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42915 2024-12-07T01:14:21,685 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42915 2024-12-07T01:14:21,686 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42915 2024-12-07T01:14:21,686 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42915 2024-12-07T01:14:21,702 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/ec1863dc21e5:0 server-side Connection retries=45 2024-12-07T01:14:21,702 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-07T01:14:21,702 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-07T01:14:21,703 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-07T01:14:21,703 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-07T01:14:21,703 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-07T01:14:21,703 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-07T01:14:21,703 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-07T01:14:21,704 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:36079 2024-12-07T01:14:21,705 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:36079 connecting to ZooKeeper ensemble=127.0.0.1:61706 2024-12-07T01:14:21,706 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-07T01:14:21,709 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-07T01:14:21,723 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:360790x0, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-07T01:14:21,723 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:360790x0, quorum=127.0.0.1:61706, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-07T01:14:21,724 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36079-0x101ad5a899b0002 connected 2024-12-07T01:14:21,724 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-07T01:14:21,725 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-07T01:14:21,726 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36079-0x101ad5a899b0002, quorum=127.0.0.1:61706, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-07T01:14:21,727 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36079-0x101ad5a899b0002, quorum=127.0.0.1:61706, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-07T01:14:21,728 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36079 2024-12-07T01:14:21,728 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36079 2024-12-07T01:14:21,729 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36079 2024-12-07T01:14:21,730 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36079 2024-12-07T01:14:21,731 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36079 2024-12-07T01:14:21,745 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/ec1863dc21e5:0 server-side Connection retries=45 2024-12-07T01:14:21,745 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-07T01:14:21,745 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-07T01:14:21,745 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-07T01:14:21,746 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-07T01:14:21,746 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-07T01:14:21,746 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-07T01:14:21,746 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-07T01:14:21,747 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:40693 2024-12-07T01:14:21,748 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40693 connecting to ZooKeeper ensemble=127.0.0.1:61706 2024-12-07T01:14:21,749 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-07T01:14:21,751 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-07T01:14:21,765 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:406930x0, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-07T01:14:21,765 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40693-0x101ad5a899b0003 connected 2024-12-07T01:14:21,765 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40693-0x101ad5a899b0003, quorum=127.0.0.1:61706, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-07T01:14:21,766 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-07T01:14:21,767 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-07T01:14:21,768 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40693-0x101ad5a899b0003, quorum=127.0.0.1:61706, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-07T01:14:21,769 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40693-0x101ad5a899b0003, quorum=127.0.0.1:61706, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-07T01:14:21,770 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40693 2024-12-07T01:14:21,770 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40693 2024-12-07T01:14:21,770 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40693 2024-12-07T01:14:21,771 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40693 2024-12-07T01:14:21,771 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40693 2024-12-07T01:14:21,787 DEBUG [M:0;ec1863dc21e5:46141 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;ec1863dc21e5:46141 2024-12-07T01:14:21,788 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/ec1863dc21e5,46141,1733534060840 2024-12-07T01:14:21,835 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40693-0x101ad5a899b0003, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-07T01:14:21,835 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46141-0x101ad5a899b0000, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-07T01:14:21,835 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36079-0x101ad5a899b0002, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-07T01:14:21,835 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42915-0x101ad5a899b0001, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-07T01:14:21,838 DEBUG [master/ec1863dc21e5:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:46141-0x101ad5a899b0000, quorum=127.0.0.1:61706, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/ec1863dc21e5,46141,1733534060840 2024-12-07T01:14:21,891 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36079-0x101ad5a899b0002, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-07T01:14:21,891 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46141-0x101ad5a899b0000, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T01:14:21,891 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40693-0x101ad5a899b0003, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-07T01:14:21,891 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42915-0x101ad5a899b0001, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-07T01:14:21,891 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40693-0x101ad5a899b0003, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T01:14:21,891 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42915-0x101ad5a899b0001, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T01:14:21,891 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36079-0x101ad5a899b0002, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T01:14:21,893 DEBUG [master/ec1863dc21e5:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:46141-0x101ad5a899b0000, quorum=127.0.0.1:61706, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-07T01:14:21,894 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/ec1863dc21e5,46141,1733534060840 from backup master directory 2024-12-07T01:14:21,907 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42915-0x101ad5a899b0001, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-07T01:14:21,907 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46141-0x101ad5a899b0000, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/ec1863dc21e5,46141,1733534060840 2024-12-07T01:14:21,907 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40693-0x101ad5a899b0003, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-07T01:14:21,907 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36079-0x101ad5a899b0002, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-07T01:14:21,907 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46141-0x101ad5a899b0000, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-07T01:14:21,908 WARN [master/ec1863dc21e5:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-07T01:14:21,909 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=ec1863dc21e5,46141,1733534060840 2024-12-07T01:14:21,911 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-07T01:14:21,913 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-07T01:14:21,962 DEBUG [master/ec1863dc21e5:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/hbase.id] with ID: 68178764-2765-461b-a758-74a88455d04a 2024-12-07T01:14:21,962 DEBUG [master/ec1863dc21e5:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/.tmp/hbase.id 2024-12-07T01:14:21,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43991 is added to blk_1073741826_1002 (size=42) 2024-12-07T01:14:21,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741826_1002 (size=42) 2024-12-07T01:14:21,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37235 is added to blk_1073741826_1002 (size=42) 2024-12-07T01:14:21,977 DEBUG [master/ec1863dc21e5:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/.tmp/hbase.id]:[hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/hbase.id] 2024-12-07T01:14:22,021 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-07T01:14:22,025 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-07T01:14:22,053 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 26ms. 2024-12-07T01:14:22,138 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42915-0x101ad5a899b0001, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T01:14:22,138 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40693-0x101ad5a899b0003, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T01:14:22,138 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36079-0x101ad5a899b0002, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T01:14:22,138 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46141-0x101ad5a899b0000, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T01:14:22,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37235 is added to blk_1073741827_1003 (size=196) 2024-12-07T01:14:22,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43991 is added to blk_1073741827_1003 (size=196) 2024-12-07T01:14:22,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741827_1003 (size=196) 2024-12-07T01:14:22,175 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-07T01:14:22,176 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-07T01:14:22,190 DEBUG [master/ec1863dc21e5:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T01:14:22,193 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-07T01:14:22,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43991 is added to blk_1073741828_1004 (size=1189) 2024-12-07T01:14:22,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37235 is added to blk_1073741828_1004 (size=1189) 2024-12-07T01:14:22,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741828_1004 (size=1189) 2024-12-07T01:14:22,235 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/MasterData/data/master/store 2024-12-07T01:14:22,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741829_1005 (size=34) 2024-12-07T01:14:22,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37235 is added to blk_1073741829_1005 (size=34) 2024-12-07T01:14:22,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43991 is added to blk_1073741829_1005 (size=34) 2024-12-07T01:14:22,259 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-07T01:14:22,262 DEBUG [master/ec1863dc21e5:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T01:14:22,263 DEBUG [master/ec1863dc21e5:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-07T01:14:22,263 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-07T01:14:22,263 DEBUG [master/ec1863dc21e5:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-07T01:14:22,265 DEBUG [master/ec1863dc21e5:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-07T01:14:22,265 DEBUG [master/ec1863dc21e5:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-07T01:14:22,265 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-07T01:14:22,266 DEBUG [master/ec1863dc21e5:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733534062263Disabling compacts and flushes for region at 1733534062263Disabling writes for close at 1733534062265 (+2 ms)Writing region close event to WAL at 1733534062265Closed at 1733534062265 2024-12-07T01:14:22,268 WARN [master/ec1863dc21e5:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/MasterData/data/master/store/.initializing 2024-12-07T01:14:22,268 DEBUG [master/ec1863dc21e5:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/MasterData/WALs/ec1863dc21e5,46141,1733534060840 2024-12-07T01:14:22,275 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-07T01:14:22,289 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ec1863dc21e5%2C46141%2C1733534060840, suffix=, logDir=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/MasterData/WALs/ec1863dc21e5,46141,1733534060840, archiveDir=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/MasterData/oldWALs, maxLogs=10 2024-12-07T01:14:22,307 DEBUG [master/ec1863dc21e5:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/MasterData/WALs/ec1863dc21e5,46141,1733534060840/ec1863dc21e5%2C46141%2C1733534060840.1733534062293, exclude list is [], retry=0 2024-12-07T01:14:22,322 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37235,DS-b9cb305a-a2e1-439c-816b-c45d44c27920,DISK] 2024-12-07T01:14:22,322 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43991,DS-f5ad7f4d-d62b-424d-bad1-dbf9e5694bea,DISK] 2024-12-07T01:14:22,322 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40127,DS-59048819-6295-483c-8edd-efcadeb66bf1,DISK] 2024-12-07T01:14:22,325 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-07T01:14:22,359 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/MasterData/WALs/ec1863dc21e5,46141,1733534060840/ec1863dc21e5%2C46141%2C1733534060840.1733534062293 2024-12-07T01:14:22,360 DEBUG [master/ec1863dc21e5:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39885:39885),(127.0.0.1/127.0.0.1:40865:40865),(127.0.0.1/127.0.0.1:45341:45341)] 2024-12-07T01:14:22,361 DEBUG [master/ec1863dc21e5:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-07T01:14:22,361 DEBUG [master/ec1863dc21e5:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T01:14:22,364 DEBUG [master/ec1863dc21e5:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-07T01:14:22,365 DEBUG [master/ec1863dc21e5:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-07T01:14:22,397 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-07T01:14:22,419 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-07T01:14:22,422 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:14:22,425 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T01:14:22,425 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-07T01:14:22,428 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-07T01:14:22,428 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:14:22,429 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T01:14:22,430 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-07T01:14:22,432 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-07T01:14:22,432 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:14:22,433 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T01:14:22,434 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-07T01:14:22,436 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-07T01:14:22,436 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:14:22,437 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T01:14:22,438 DEBUG [master/ec1863dc21e5:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-07T01:14:22,442 DEBUG [master/ec1863dc21e5:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-07T01:14:22,443 DEBUG [master/ec1863dc21e5:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-07T01:14:22,449 DEBUG [master/ec1863dc21e5:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-07T01:14:22,449 DEBUG [master/ec1863dc21e5:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-07T01:14:22,453 DEBUG [master/ec1863dc21e5:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-07T01:14:22,456 DEBUG [master/ec1863dc21e5:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-07T01:14:22,460 DEBUG [master/ec1863dc21e5:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-07T01:14:22,462 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62991470, jitterRate=-0.06135395169258118}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-07T01:14:22,467 DEBUG [master/ec1863dc21e5:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733534062376Initializing all the Stores at 1733534062378 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733534062378Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534062379 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534062379Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534062379Cleaning up temporary data from old regions at 1733534062450 (+71 ms)Region opened successfully at 1733534062466 (+16 ms) 2024-12-07T01:14:22,468 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-07T01:14:22,499 DEBUG [master/ec1863dc21e5:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e7f4b98, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ec1863dc21e5/172.17.0.3:0 2024-12-07T01:14:22,525 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-07T01:14:22,535 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-07T01:14:22,535 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-07T01:14:22,537 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-07T01:14:22,538 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-07T01:14:22,542 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-12-07T01:14:22,543 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-07T01:14:22,565 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-07T01:14:22,573 DEBUG [master/ec1863dc21e5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46141-0x101ad5a899b0000, quorum=127.0.0.1:61706, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-07T01:14:22,659 DEBUG [master/ec1863dc21e5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-07T01:14:22,662 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-07T01:14:22,663 DEBUG [master/ec1863dc21e5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46141-0x101ad5a899b0000, quorum=127.0.0.1:61706, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-07T01:14:22,722 DEBUG [master/ec1863dc21e5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-07T01:14:22,725 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-07T01:14:22,728 DEBUG [master/ec1863dc21e5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46141-0x101ad5a899b0000, quorum=127.0.0.1:61706, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-07T01:14:22,738 DEBUG [master/ec1863dc21e5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-07T01:14:22,739 DEBUG [master/ec1863dc21e5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46141-0x101ad5a899b0000, quorum=127.0.0.1:61706, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-07T01:14:22,748 DEBUG [master/ec1863dc21e5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-07T01:14:22,765 DEBUG [master/ec1863dc21e5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46141-0x101ad5a899b0000, quorum=127.0.0.1:61706, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-07T01:14:22,775 DEBUG [master/ec1863dc21e5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-07T01:14:22,786 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36079-0x101ad5a899b0002, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-07T01:14:22,786 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42915-0x101ad5a899b0001, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-07T01:14:22,786 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46141-0x101ad5a899b0000, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-07T01:14:22,786 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40693-0x101ad5a899b0003, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-07T01:14:22,786 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42915-0x101ad5a899b0001, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T01:14:22,786 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46141-0x101ad5a899b0000, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T01:14:22,786 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40693-0x101ad5a899b0003, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T01:14:22,787 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36079-0x101ad5a899b0002, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T01:14:22,791 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=ec1863dc21e5,46141,1733534060840, sessionid=0x101ad5a899b0000, setting cluster-up flag (Was=false) 2024-12-07T01:14:22,822 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40693-0x101ad5a899b0003, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T01:14:22,822 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46141-0x101ad5a899b0000, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T01:14:22,822 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36079-0x101ad5a899b0002, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T01:14:22,822 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42915-0x101ad5a899b0001, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T01:14:22,855 DEBUG [master/ec1863dc21e5:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-07T01:14:22,859 DEBUG [master/ec1863dc21e5:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=ec1863dc21e5,46141,1733534060840 2024-12-07T01:14:22,886 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36079-0x101ad5a899b0002, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T01:14:22,886 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40693-0x101ad5a899b0003, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T01:14:22,886 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46141-0x101ad5a899b0000, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T01:14:22,886 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42915-0x101ad5a899b0001, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T01:14:22,922 DEBUG [master/ec1863dc21e5:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-07T01:14:22,925 DEBUG [master/ec1863dc21e5:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=ec1863dc21e5,46141,1733534060840 2024-12-07T01:14:22,933 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-07T01:14:22,975 INFO [RS:1;ec1863dc21e5:36079 {}] regionserver.HRegionServer(746): ClusterId : 68178764-2765-461b-a758-74a88455d04a 2024-12-07T01:14:22,975 INFO [RS:2;ec1863dc21e5:40693 {}] regionserver.HRegionServer(746): ClusterId : 68178764-2765-461b-a758-74a88455d04a 2024-12-07T01:14:22,975 INFO [RS:0;ec1863dc21e5:42915 {}] regionserver.HRegionServer(746): ClusterId : 68178764-2765-461b-a758-74a88455d04a 2024-12-07T01:14:22,978 DEBUG [RS:0;ec1863dc21e5:42915 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-07T01:14:22,978 DEBUG [RS:2;ec1863dc21e5:40693 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-07T01:14:22,978 DEBUG [RS:1;ec1863dc21e5:36079 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-07T01:14:22,993 DEBUG [RS:2;ec1863dc21e5:40693 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-07T01:14:22,993 DEBUG [RS:1;ec1863dc21e5:36079 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-07T01:14:22,993 DEBUG [RS:0;ec1863dc21e5:42915 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-07T01:14:22,993 DEBUG [RS:2;ec1863dc21e5:40693 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-07T01:14:22,993 DEBUG [RS:1;ec1863dc21e5:36079 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-07T01:14:22,993 DEBUG [RS:0;ec1863dc21e5:42915 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-07T01:14:23,003 DEBUG [RS:2;ec1863dc21e5:40693 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-07T01:14:23,003 DEBUG [RS:0;ec1863dc21e5:42915 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-07T01:14:23,003 DEBUG [RS:1;ec1863dc21e5:36079 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-07T01:14:23,002 DEBUG [master/ec1863dc21e5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-07T01:14:23,003 DEBUG [RS:2;ec1863dc21e5:40693 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ad4e77a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ec1863dc21e5/172.17.0.3:0 2024-12-07T01:14:23,003 DEBUG [RS:1;ec1863dc21e5:36079 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69ccd94b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ec1863dc21e5/172.17.0.3:0 2024-12-07T01:14:23,003 DEBUG [RS:0;ec1863dc21e5:42915 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@383648f8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ec1863dc21e5/172.17.0.3:0 2024-12-07T01:14:23,014 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-07T01:14:23,016 DEBUG [RS:0;ec1863dc21e5:42915 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;ec1863dc21e5:42915 2024-12-07T01:14:23,019 INFO [RS:0;ec1863dc21e5:42915 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-07T01:14:23,020 INFO [RS:0;ec1863dc21e5:42915 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-07T01:14:23,020 DEBUG [RS:0;ec1863dc21e5:42915 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-07T01:14:23,022 DEBUG [RS:2;ec1863dc21e5:40693 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;ec1863dc21e5:40693 2024-12-07T01:14:23,022 DEBUG [RS:1;ec1863dc21e5:36079 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;ec1863dc21e5:36079 2024-12-07T01:14:23,022 INFO [RS:0;ec1863dc21e5:42915 {}] regionserver.HRegionServer(2659): reportForDuty to master=ec1863dc21e5,46141,1733534060840 with port=42915, startcode=1733534061580 2024-12-07T01:14:23,022 INFO [RS:1;ec1863dc21e5:36079 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-07T01:14:23,022 INFO [RS:2;ec1863dc21e5:40693 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-07T01:14:23,022 INFO [RS:1;ec1863dc21e5:36079 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-07T01:14:23,022 INFO [RS:2;ec1863dc21e5:40693 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-07T01:14:23,022 DEBUG [RS:1;ec1863dc21e5:36079 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-07T01:14:23,022 DEBUG [RS:2;ec1863dc21e5:40693 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-07T01:14:23,023 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-07T01:14:23,024 INFO [RS:1;ec1863dc21e5:36079 {}] regionserver.HRegionServer(2659): reportForDuty to master=ec1863dc21e5,46141,1733534060840 with port=36079, startcode=1733534061701 2024-12-07T01:14:23,024 INFO [RS:2;ec1863dc21e5:40693 {}] regionserver.HRegionServer(2659): reportForDuty to master=ec1863dc21e5,46141,1733534060840 with port=40693, startcode=1733534061744 2024-12-07T01:14:23,029 DEBUG [master/ec1863dc21e5:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: ec1863dc21e5,46141,1733534060840 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-07T01:14:23,033 DEBUG [RS:2;ec1863dc21e5:40693 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-07T01:14:23,033 DEBUG [RS:1;ec1863dc21e5:36079 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-07T01:14:23,033 DEBUG [RS:0;ec1863dc21e5:42915 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-07T01:14:23,036 DEBUG [master/ec1863dc21e5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/ec1863dc21e5:0, corePoolSize=5, maxPoolSize=5 2024-12-07T01:14:23,036 DEBUG [master/ec1863dc21e5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/ec1863dc21e5:0, corePoolSize=5, maxPoolSize=5 2024-12-07T01:14:23,037 DEBUG [master/ec1863dc21e5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/ec1863dc21e5:0, corePoolSize=5, maxPoolSize=5 2024-12-07T01:14:23,037 DEBUG [master/ec1863dc21e5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/ec1863dc21e5:0, corePoolSize=5, maxPoolSize=5 2024-12-07T01:14:23,037 DEBUG [master/ec1863dc21e5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/ec1863dc21e5:0, corePoolSize=10, maxPoolSize=10 2024-12-07T01:14:23,037 DEBUG [master/ec1863dc21e5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/ec1863dc21e5:0, corePoolSize=1, maxPoolSize=1 2024-12-07T01:14:23,037 DEBUG [master/ec1863dc21e5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/ec1863dc21e5:0, corePoolSize=2, maxPoolSize=2 2024-12-07T01:14:23,038 DEBUG [master/ec1863dc21e5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/ec1863dc21e5:0, corePoolSize=1, maxPoolSize=1 2024-12-07T01:14:23,039 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733534093039 2024-12-07T01:14:23,041 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-07T01:14:23,042 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-07T01:14:23,043 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-07T01:14:23,043 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-07T01:14:23,046 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-07T01:14:23,047 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-07T01:14:23,047 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-07T01:14:23,048 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-07T01:14:23,050 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:14:23,050 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-07T01:14:23,056 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-07T01:14:23,061 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-07T01:14:23,062 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-07T01:14:23,063 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-07T01:14:23,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43991 is added to blk_1073741831_1007 (size=1321) 2024-12-07T01:14:23,106 INFO [HMasterForTest-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58873, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-07T01:14:23,106 INFO [HMasterForTest-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38791, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-12-07T01:14:23,106 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-07T01:14:23,107 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-07T01:14:23,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37235 is added to blk_1073741831_1007 (size=1321) 2024-12-07T01:14:23,107 INFO [HMasterForTest-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48793, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-12-07T01:14:23,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741831_1007 (size=1321) 2024-12-07T01:14:23,109 DEBUG [master/ec1863dc21e5:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/ec1863dc21e5:0:becomeActiveMaster-HFileCleaner.large.0-1733534063109,5,FailOnTimeoutGroup] 2024-12-07T01:14:23,110 DEBUG [master/ec1863dc21e5:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/ec1863dc21e5:0:becomeActiveMaster-HFileCleaner.small.0-1733534063110,5,FailOnTimeoutGroup] 2024-12-07T01:14:23,110 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-07T01:14:23,110 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-07T01:14:23,110 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-07T01:14:23,111 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323 2024-12-07T01:14:23,111 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-07T01:14:23,112 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-07T01:14:23,114 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46141 {}] master.ServerManager(363): Checking decommissioned status of RegionServer ec1863dc21e5,36079,1733534061701 2024-12-07T01:14:23,117 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46141 {}] master.ServerManager(517): Registering regionserver=ec1863dc21e5,36079,1733534061701 2024-12-07T01:14:23,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741832_1008 (size=32) 2024-12-07T01:14:23,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37235 is added to blk_1073741832_1008 (size=32) 2024-12-07T01:14:23,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43991 is added to blk_1073741832_1008 (size=32) 2024-12-07T01:14:23,125 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T01:14:23,127 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-07T01:14:23,130 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46141 {}] master.ServerManager(363): Checking decommissioned status of RegionServer ec1863dc21e5,40693,1733534061744 2024-12-07T01:14:23,130 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46141 {}] master.ServerManager(517): Registering regionserver=ec1863dc21e5,40693,1733534061744 2024-12-07T01:14:23,130 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-07T01:14:23,131 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:14:23,132 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T01:14:23,132 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-07T01:14:23,133 DEBUG [RS:1;ec1863dc21e5:36079 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323 2024-12-07T01:14:23,133 DEBUG [RS:1;ec1863dc21e5:36079 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:35877 2024-12-07T01:14:23,133 DEBUG [RS:1;ec1863dc21e5:36079 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-07T01:14:23,135 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-07T01:14:23,135 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:14:23,137 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46141 {}] master.ServerManager(363): Checking decommissioned status of RegionServer ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:23,137 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46141 {}] master.ServerManager(517): Registering regionserver=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:23,137 DEBUG [RS:2;ec1863dc21e5:40693 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323 2024-12-07T01:14:23,137 DEBUG [RS:2;ec1863dc21e5:40693 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:35877 2024-12-07T01:14:23,137 DEBUG [RS:2;ec1863dc21e5:40693 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-07T01:14:23,140 DEBUG [RS:0;ec1863dc21e5:42915 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323 2024-12-07T01:14:23,140 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T01:14:23,140 DEBUG [RS:0;ec1863dc21e5:42915 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:35877 2024-12-07T01:14:23,140 DEBUG [RS:0;ec1863dc21e5:42915 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-07T01:14:23,140 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-07T01:14:23,143 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-07T01:14:23,143 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:14:23,144 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T01:14:23,144 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-07T01:14:23,147 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-07T01:14:23,147 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:14:23,148 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T01:14:23,148 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-07T01:14:23,149 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46141-0x101ad5a899b0000, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-07T01:14:23,150 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/hbase/meta/1588230740 2024-12-07T01:14:23,151 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/hbase/meta/1588230740 2024-12-07T01:14:23,153 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-07T01:14:23,153 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-07T01:14:23,154 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-07T01:14:23,157 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-07T01:14:23,161 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-07T01:14:23,162 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59371343, jitterRate=-0.11529804766178131}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-07T01:14:23,164 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733534063126Initializing all the Stores at 1733534063127 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733534063127Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733534063127Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534063127Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733534063127Cleaning up temporary data from old regions at 1733534063153 (+26 ms)Region opened successfully at 1733534063164 (+11 ms) 2024-12-07T01:14:23,164 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-07T01:14:23,165 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-07T01:14:23,165 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-07T01:14:23,165 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-07T01:14:23,165 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-07T01:14:23,166 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-07T01:14:23,166 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733534063164Disabling compacts and flushes for region at 1733534063164Disabling writes for close at 1733534063165 (+1 ms)Writing region close event to WAL at 1733534063166 (+1 ms)Closed at 1733534063166 2024-12-07T01:14:23,169 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-07T01:14:23,169 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-07T01:14:23,175 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-07T01:14:23,184 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-07T01:14:23,187 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-07T01:14:23,222 DEBUG [RS:1;ec1863dc21e5:36079 {}] zookeeper.ZKUtil(111): regionserver:36079-0x101ad5a899b0002, quorum=127.0.0.1:61706, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/ec1863dc21e5,36079,1733534061701 2024-12-07T01:14:23,222 WARN [RS:1;ec1863dc21e5:36079 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-07T01:14:23,223 INFO [RS:1;ec1863dc21e5:36079 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-07T01:14:23,223 DEBUG [RS:2;ec1863dc21e5:40693 {}] zookeeper.ZKUtil(111): regionserver:40693-0x101ad5a899b0003, quorum=127.0.0.1:61706, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/ec1863dc21e5,40693,1733534061744 2024-12-07T01:14:23,223 DEBUG [RS:0;ec1863dc21e5:42915 {}] zookeeper.ZKUtil(111): regionserver:42915-0x101ad5a899b0001, quorum=127.0.0.1:61706, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:23,223 WARN [RS:0;ec1863dc21e5:42915 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-07T01:14:23,223 WARN [RS:2;ec1863dc21e5:40693 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-07T01:14:23,223 DEBUG [RS:1;ec1863dc21e5:36079 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/WALs/ec1863dc21e5,36079,1733534061701 2024-12-07T01:14:23,223 INFO [RS:0;ec1863dc21e5:42915 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-07T01:14:23,223 INFO [RS:2;ec1863dc21e5:40693 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-07T01:14:23,223 DEBUG [RS:0;ec1863dc21e5:42915 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/WALs/ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:23,223 DEBUG [RS:2;ec1863dc21e5:40693 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/WALs/ec1863dc21e5,40693,1733534061744 2024-12-07T01:14:23,224 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [ec1863dc21e5,36079,1733534061701] 2024-12-07T01:14:23,224 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [ec1863dc21e5,40693,1733534061744] 2024-12-07T01:14:23,224 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [ec1863dc21e5,42915,1733534061580] 2024-12-07T01:14:23,254 INFO [RS:0;ec1863dc21e5:42915 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-07T01:14:23,254 INFO [RS:2;ec1863dc21e5:40693 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-07T01:14:23,254 INFO [RS:1;ec1863dc21e5:36079 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-07T01:14:23,269 INFO [RS:1;ec1863dc21e5:36079 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-07T01:14:23,269 INFO [RS:0;ec1863dc21e5:42915 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-07T01:14:23,269 INFO [RS:2;ec1863dc21e5:40693 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-07T01:14:23,275 INFO [RS:0;ec1863dc21e5:42915 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-07T01:14:23,275 INFO [RS:2;ec1863dc21e5:40693 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-07T01:14:23,275 INFO [RS:1;ec1863dc21e5:36079 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-07T01:14:23,275 INFO [RS:0;ec1863dc21e5:42915 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-07T01:14:23,275 INFO [RS:1;ec1863dc21e5:36079 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-07T01:14:23,275 INFO [RS:2;ec1863dc21e5:40693 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-07T01:14:23,276 INFO [RS:2;ec1863dc21e5:40693 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-07T01:14:23,276 INFO [RS:1;ec1863dc21e5:36079 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-07T01:14:23,276 INFO [RS:0;ec1863dc21e5:42915 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-07T01:14:23,282 INFO [RS:0;ec1863dc21e5:42915 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-07T01:14:23,282 INFO [RS:1;ec1863dc21e5:36079 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-07T01:14:23,282 INFO [RS:2;ec1863dc21e5:40693 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-07T01:14:23,284 INFO [RS:0;ec1863dc21e5:42915 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-07T01:14:23,284 INFO [RS:1;ec1863dc21e5:36079 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-07T01:14:23,284 DEBUG [RS:0;ec1863dc21e5:42915 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/ec1863dc21e5:0, corePoolSize=1, maxPoolSize=1 2024-12-07T01:14:23,284 DEBUG [RS:1;ec1863dc21e5:36079 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/ec1863dc21e5:0, corePoolSize=1, maxPoolSize=1 2024-12-07T01:14:23,284 INFO [RS:2;ec1863dc21e5:40693 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-07T01:14:23,284 DEBUG [RS:1;ec1863dc21e5:36079 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/ec1863dc21e5:0, corePoolSize=1, maxPoolSize=1 2024-12-07T01:14:23,284 DEBUG [RS:0;ec1863dc21e5:42915 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/ec1863dc21e5:0, corePoolSize=1, maxPoolSize=1 2024-12-07T01:14:23,285 DEBUG [RS:1;ec1863dc21e5:36079 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/ec1863dc21e5:0, corePoolSize=1, maxPoolSize=1 2024-12-07T01:14:23,285 DEBUG [RS:0;ec1863dc21e5:42915 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/ec1863dc21e5:0, corePoolSize=1, maxPoolSize=1 2024-12-07T01:14:23,285 DEBUG [RS:1;ec1863dc21e5:36079 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/ec1863dc21e5:0, corePoolSize=1, maxPoolSize=1 2024-12-07T01:14:23,285 DEBUG [RS:0;ec1863dc21e5:42915 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/ec1863dc21e5:0, corePoolSize=1, maxPoolSize=1 2024-12-07T01:14:23,285 DEBUG [RS:2;ec1863dc21e5:40693 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/ec1863dc21e5:0, corePoolSize=1, maxPoolSize=1 2024-12-07T01:14:23,285 DEBUG [RS:1;ec1863dc21e5:36079 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/ec1863dc21e5:0, corePoolSize=1, maxPoolSize=1 2024-12-07T01:14:23,285 DEBUG [RS:0;ec1863dc21e5:42915 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/ec1863dc21e5:0, corePoolSize=1, maxPoolSize=1 2024-12-07T01:14:23,285 DEBUG [RS:1;ec1863dc21e5:36079 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/ec1863dc21e5:0, corePoolSize=2, maxPoolSize=2 2024-12-07T01:14:23,285 DEBUG [RS:1;ec1863dc21e5:36079 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/ec1863dc21e5:0, corePoolSize=1, maxPoolSize=1 2024-12-07T01:14:23,285 DEBUG [RS:2;ec1863dc21e5:40693 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/ec1863dc21e5:0, corePoolSize=1, maxPoolSize=1 2024-12-07T01:14:23,285 DEBUG [RS:0;ec1863dc21e5:42915 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/ec1863dc21e5:0, corePoolSize=2, maxPoolSize=2 2024-12-07T01:14:23,285 DEBUG [RS:1;ec1863dc21e5:36079 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/ec1863dc21e5:0, corePoolSize=1, maxPoolSize=1 2024-12-07T01:14:23,285 DEBUG [RS:0;ec1863dc21e5:42915 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/ec1863dc21e5:0, corePoolSize=1, maxPoolSize=1 2024-12-07T01:14:23,285 DEBUG [RS:1;ec1863dc21e5:36079 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/ec1863dc21e5:0, corePoolSize=1, maxPoolSize=1 2024-12-07T01:14:23,285 DEBUG [RS:2;ec1863dc21e5:40693 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/ec1863dc21e5:0, corePoolSize=1, maxPoolSize=1 2024-12-07T01:14:23,285 DEBUG [RS:1;ec1863dc21e5:36079 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/ec1863dc21e5:0, corePoolSize=1, maxPoolSize=1 2024-12-07T01:14:23,285 DEBUG [RS:0;ec1863dc21e5:42915 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/ec1863dc21e5:0, corePoolSize=1, maxPoolSize=1 2024-12-07T01:14:23,285 DEBUG [RS:1;ec1863dc21e5:36079 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/ec1863dc21e5:0, corePoolSize=1, maxPoolSize=1 2024-12-07T01:14:23,285 DEBUG [RS:0;ec1863dc21e5:42915 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/ec1863dc21e5:0, corePoolSize=1, maxPoolSize=1 2024-12-07T01:14:23,286 DEBUG [RS:1;ec1863dc21e5:36079 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/ec1863dc21e5:0, corePoolSize=1, maxPoolSize=1 2024-12-07T01:14:23,286 DEBUG [RS:2;ec1863dc21e5:40693 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/ec1863dc21e5:0, corePoolSize=1, maxPoolSize=1 2024-12-07T01:14:23,286 DEBUG [RS:1;ec1863dc21e5:36079 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/ec1863dc21e5:0, corePoolSize=3, maxPoolSize=3 2024-12-07T01:14:23,286 DEBUG [RS:0;ec1863dc21e5:42915 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/ec1863dc21e5:0, corePoolSize=1, maxPoolSize=1 2024-12-07T01:14:23,286 DEBUG [RS:1;ec1863dc21e5:36079 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/ec1863dc21e5:0, corePoolSize=3, maxPoolSize=3 2024-12-07T01:14:23,286 DEBUG [RS:0;ec1863dc21e5:42915 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/ec1863dc21e5:0, corePoolSize=1, maxPoolSize=1 2024-12-07T01:14:23,286 DEBUG [RS:2;ec1863dc21e5:40693 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/ec1863dc21e5:0, corePoolSize=1, maxPoolSize=1 2024-12-07T01:14:23,286 DEBUG [RS:0;ec1863dc21e5:42915 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/ec1863dc21e5:0, corePoolSize=1, maxPoolSize=1 2024-12-07T01:14:23,286 DEBUG [RS:0;ec1863dc21e5:42915 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/ec1863dc21e5:0, corePoolSize=3, maxPoolSize=3 2024-12-07T01:14:23,286 DEBUG [RS:2;ec1863dc21e5:40693 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/ec1863dc21e5:0, corePoolSize=2, maxPoolSize=2 2024-12-07T01:14:23,286 DEBUG [RS:0;ec1863dc21e5:42915 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/ec1863dc21e5:0, corePoolSize=3, maxPoolSize=3 2024-12-07T01:14:23,286 DEBUG [RS:2;ec1863dc21e5:40693 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/ec1863dc21e5:0, corePoolSize=1, maxPoolSize=1 2024-12-07T01:14:23,287 DEBUG [RS:2;ec1863dc21e5:40693 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/ec1863dc21e5:0, corePoolSize=1, maxPoolSize=1 2024-12-07T01:14:23,287 INFO [RS:1;ec1863dc21e5:36079 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-07T01:14:23,287 INFO [RS:1;ec1863dc21e5:36079 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-07T01:14:23,287 DEBUG [RS:2;ec1863dc21e5:40693 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/ec1863dc21e5:0, corePoolSize=1, maxPoolSize=1 2024-12-07T01:14:23,287 INFO [RS:1;ec1863dc21e5:36079 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-07T01:14:23,287 INFO [RS:1;ec1863dc21e5:36079 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-07T01:14:23,287 INFO [RS:1;ec1863dc21e5:36079 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-07T01:14:23,287 INFO [RS:1;ec1863dc21e5:36079 {}] hbase.ChoreService(168): Chore ScheduledChore name=ec1863dc21e5,36079,1733534061701-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-07T01:14:23,287 DEBUG [RS:2;ec1863dc21e5:40693 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/ec1863dc21e5:0, corePoolSize=1, maxPoolSize=1 2024-12-07T01:14:23,288 DEBUG [RS:2;ec1863dc21e5:40693 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/ec1863dc21e5:0, corePoolSize=1, maxPoolSize=1 2024-12-07T01:14:23,288 INFO [RS:0;ec1863dc21e5:42915 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-07T01:14:23,288 INFO [RS:0;ec1863dc21e5:42915 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-07T01:14:23,288 INFO [RS:0;ec1863dc21e5:42915 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-07T01:14:23,288 INFO [RS:0;ec1863dc21e5:42915 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-07T01:14:23,288 INFO [RS:0;ec1863dc21e5:42915 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-07T01:14:23,288 DEBUG [RS:2;ec1863dc21e5:40693 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/ec1863dc21e5:0, corePoolSize=1, maxPoolSize=1 2024-12-07T01:14:23,288 INFO [RS:0;ec1863dc21e5:42915 {}] hbase.ChoreService(168): Chore ScheduledChore name=ec1863dc21e5,42915,1733534061580-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-07T01:14:23,289 DEBUG [RS:2;ec1863dc21e5:40693 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/ec1863dc21e5:0, corePoolSize=3, maxPoolSize=3 2024-12-07T01:14:23,289 DEBUG [RS:2;ec1863dc21e5:40693 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/ec1863dc21e5:0, corePoolSize=3, maxPoolSize=3 2024-12-07T01:14:23,296 INFO [RS:2;ec1863dc21e5:40693 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-07T01:14:23,297 INFO [RS:2;ec1863dc21e5:40693 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-07T01:14:23,297 INFO [RS:2;ec1863dc21e5:40693 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-07T01:14:23,297 INFO [RS:2;ec1863dc21e5:40693 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-07T01:14:23,297 INFO [RS:2;ec1863dc21e5:40693 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-07T01:14:23,297 INFO [RS:2;ec1863dc21e5:40693 {}] hbase.ChoreService(168): Chore ScheduledChore name=ec1863dc21e5,40693,1733534061744-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-07T01:14:23,315 INFO [RS:2;ec1863dc21e5:40693 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-07T01:14:23,315 INFO [RS:0;ec1863dc21e5:42915 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-07T01:14:23,315 INFO [RS:1;ec1863dc21e5:36079 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-07T01:14:23,317 INFO [RS:2;ec1863dc21e5:40693 {}] hbase.ChoreService(168): Chore ScheduledChore name=ec1863dc21e5,40693,1733534061744-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-07T01:14:23,317 INFO [RS:0;ec1863dc21e5:42915 {}] hbase.ChoreService(168): Chore ScheduledChore name=ec1863dc21e5,42915,1733534061580-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-07T01:14:23,317 INFO [RS:1;ec1863dc21e5:36079 {}] hbase.ChoreService(168): Chore ScheduledChore name=ec1863dc21e5,36079,1733534061701-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-07T01:14:23,317 INFO [RS:2;ec1863dc21e5:40693 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-07T01:14:23,317 INFO [RS:0;ec1863dc21e5:42915 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-07T01:14:23,317 INFO [RS:1;ec1863dc21e5:36079 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-07T01:14:23,317 INFO [RS:2;ec1863dc21e5:40693 {}] regionserver.Replication(171): ec1863dc21e5,40693,1733534061744 started 2024-12-07T01:14:23,317 INFO [RS:0;ec1863dc21e5:42915 {}] regionserver.Replication(171): ec1863dc21e5,42915,1733534061580 started 2024-12-07T01:14:23,317 INFO [RS:1;ec1863dc21e5:36079 {}] regionserver.Replication(171): ec1863dc21e5,36079,1733534061701 started 2024-12-07T01:14:23,333 INFO [RS:2;ec1863dc21e5:40693 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-07T01:14:23,333 INFO [RS:2;ec1863dc21e5:40693 {}] regionserver.HRegionServer(1482): Serving as ec1863dc21e5,40693,1733534061744, RpcServer on ec1863dc21e5/172.17.0.3:40693, sessionid=0x101ad5a899b0003 2024-12-07T01:14:23,334 DEBUG [RS:2;ec1863dc21e5:40693 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-07T01:14:23,334 DEBUG [RS:2;ec1863dc21e5:40693 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager ec1863dc21e5,40693,1733534061744 2024-12-07T01:14:23,334 DEBUG [RS:2;ec1863dc21e5:40693 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ec1863dc21e5,40693,1733534061744' 2024-12-07T01:14:23,334 DEBUG [RS:2;ec1863dc21e5:40693 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-07T01:14:23,335 DEBUG [RS:2;ec1863dc21e5:40693 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-07T01:14:23,335 DEBUG [RS:2;ec1863dc21e5:40693 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-07T01:14:23,335 DEBUG [RS:2;ec1863dc21e5:40693 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-07T01:14:23,335 DEBUG [RS:2;ec1863dc21e5:40693 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager ec1863dc21e5,40693,1733534061744 2024-12-07T01:14:23,336 DEBUG [RS:2;ec1863dc21e5:40693 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ec1863dc21e5,40693,1733534061744' 2024-12-07T01:14:23,336 DEBUG [RS:2;ec1863dc21e5:40693 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-07T01:14:23,336 DEBUG [RS:2;ec1863dc21e5:40693 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-07T01:14:23,336 DEBUG [RS:2;ec1863dc21e5:40693 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-07T01:14:23,337 INFO [RS:2;ec1863dc21e5:40693 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-07T01:14:23,337 INFO [RS:2;ec1863dc21e5:40693 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-07T01:14:23,338 WARN [ec1863dc21e5:46141 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-07T01:14:23,339 INFO [RS:1;ec1863dc21e5:36079 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-07T01:14:23,339 INFO [RS:0;ec1863dc21e5:42915 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-07T01:14:23,340 INFO [RS:1;ec1863dc21e5:36079 {}] regionserver.HRegionServer(1482): Serving as ec1863dc21e5,36079,1733534061701, RpcServer on ec1863dc21e5/172.17.0.3:36079, sessionid=0x101ad5a899b0002 2024-12-07T01:14:23,340 INFO [RS:0;ec1863dc21e5:42915 {}] regionserver.HRegionServer(1482): Serving as ec1863dc21e5,42915,1733534061580, RpcServer on ec1863dc21e5/172.17.0.3:42915, sessionid=0x101ad5a899b0001 2024-12-07T01:14:23,340 DEBUG [RS:0;ec1863dc21e5:42915 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-07T01:14:23,340 DEBUG [RS:1;ec1863dc21e5:36079 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-07T01:14:23,340 DEBUG [RS:0;ec1863dc21e5:42915 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:23,340 DEBUG [RS:1;ec1863dc21e5:36079 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager ec1863dc21e5,36079,1733534061701 2024-12-07T01:14:23,340 DEBUG [RS:1;ec1863dc21e5:36079 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ec1863dc21e5,36079,1733534061701' 2024-12-07T01:14:23,340 DEBUG [RS:0;ec1863dc21e5:42915 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ec1863dc21e5,42915,1733534061580' 2024-12-07T01:14:23,340 DEBUG [RS:0;ec1863dc21e5:42915 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-07T01:14:23,340 DEBUG [RS:1;ec1863dc21e5:36079 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-07T01:14:23,341 DEBUG [RS:1;ec1863dc21e5:36079 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-07T01:14:23,341 DEBUG [RS:0;ec1863dc21e5:42915 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-07T01:14:23,341 DEBUG [RS:1;ec1863dc21e5:36079 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-07T01:14:23,341 DEBUG [RS:0;ec1863dc21e5:42915 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-07T01:14:23,341 DEBUG [RS:1;ec1863dc21e5:36079 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-07T01:14:23,341 DEBUG [RS:0;ec1863dc21e5:42915 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-07T01:14:23,342 DEBUG [RS:1;ec1863dc21e5:36079 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager ec1863dc21e5,36079,1733534061701 2024-12-07T01:14:23,342 DEBUG [RS:0;ec1863dc21e5:42915 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:23,342 DEBUG [RS:0;ec1863dc21e5:42915 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ec1863dc21e5,42915,1733534061580' 2024-12-07T01:14:23,342 DEBUG [RS:1;ec1863dc21e5:36079 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ec1863dc21e5,36079,1733534061701' 2024-12-07T01:14:23,342 DEBUG [RS:0;ec1863dc21e5:42915 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-07T01:14:23,342 DEBUG [RS:1;ec1863dc21e5:36079 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-07T01:14:23,342 DEBUG [RS:0;ec1863dc21e5:42915 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-07T01:14:23,342 DEBUG [RS:1;ec1863dc21e5:36079 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-07T01:14:23,343 DEBUG [RS:1;ec1863dc21e5:36079 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-07T01:14:23,343 DEBUG [RS:0;ec1863dc21e5:42915 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-07T01:14:23,343 INFO [RS:1;ec1863dc21e5:36079 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-07T01:14:23,343 INFO [RS:0;ec1863dc21e5:42915 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-07T01:14:23,343 INFO [RS:1;ec1863dc21e5:36079 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-07T01:14:23,343 INFO [RS:0;ec1863dc21e5:42915 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-07T01:14:23,441 INFO [RS:2;ec1863dc21e5:40693 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-07T01:14:23,443 INFO [RS:1;ec1863dc21e5:36079 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-07T01:14:23,444 INFO [RS:0;ec1863dc21e5:42915 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-07T01:14:23,444 INFO [RS:2;ec1863dc21e5:40693 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ec1863dc21e5%2C40693%2C1733534061744, suffix=, logDir=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/WALs/ec1863dc21e5,40693,1733534061744, archiveDir=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/oldWALs, maxLogs=32 2024-12-07T01:14:23,446 INFO [RS:0;ec1863dc21e5:42915 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ec1863dc21e5%2C42915%2C1733534061580, suffix=, logDir=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/WALs/ec1863dc21e5,42915,1733534061580, archiveDir=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/oldWALs, maxLogs=32 2024-12-07T01:14:23,446 INFO [RS:1;ec1863dc21e5:36079 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ec1863dc21e5%2C36079%2C1733534061701, suffix=, logDir=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/WALs/ec1863dc21e5,36079,1733534061701, archiveDir=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/oldWALs, maxLogs=32 2024-12-07T01:14:23,459 DEBUG [RS:2;ec1863dc21e5:40693 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/WALs/ec1863dc21e5,40693,1733534061744/ec1863dc21e5%2C40693%2C1733534061744.1733534063446, exclude list is [], retry=0 2024-12-07T01:14:23,463 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43991,DS-f5ad7f4d-d62b-424d-bad1-dbf9e5694bea,DISK] 2024-12-07T01:14:23,463 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37235,DS-b9cb305a-a2e1-439c-816b-c45d44c27920,DISK] 2024-12-07T01:14:23,463 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40127,DS-59048819-6295-483c-8edd-efcadeb66bf1,DISK] 2024-12-07T01:14:23,466 DEBUG [RS:1;ec1863dc21e5:36079 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/WALs/ec1863dc21e5,36079,1733534061701/ec1863dc21e5%2C36079%2C1733534061701.1733534063448, exclude list is [], retry=0 2024-12-07T01:14:23,466 DEBUG [RS:0;ec1863dc21e5:42915 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/WALs/ec1863dc21e5,42915,1733534061580/ec1863dc21e5%2C42915%2C1733534061580.1733534063448, exclude list is [], retry=0 2024-12-07T01:14:23,468 INFO [RS:2;ec1863dc21e5:40693 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/WALs/ec1863dc21e5,40693,1733534061744/ec1863dc21e5%2C40693%2C1733534061744.1733534063446 2024-12-07T01:14:23,468 DEBUG [RS:2;ec1863dc21e5:40693 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39885:39885),(127.0.0.1/127.0.0.1:40865:40865),(127.0.0.1/127.0.0.1:45341:45341)] 2024-12-07T01:14:23,470 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43991,DS-f5ad7f4d-d62b-424d-bad1-dbf9e5694bea,DISK] 2024-12-07T01:14:23,470 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43991,DS-f5ad7f4d-d62b-424d-bad1-dbf9e5694bea,DISK] 2024-12-07T01:14:23,470 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37235,DS-b9cb305a-a2e1-439c-816b-c45d44c27920,DISK] 2024-12-07T01:14:23,471 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37235,DS-b9cb305a-a2e1-439c-816b-c45d44c27920,DISK] 2024-12-07T01:14:23,471 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40127,DS-59048819-6295-483c-8edd-efcadeb66bf1,DISK] 2024-12-07T01:14:23,471 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40127,DS-59048819-6295-483c-8edd-efcadeb66bf1,DISK] 2024-12-07T01:14:23,482 INFO [RS:1;ec1863dc21e5:36079 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/WALs/ec1863dc21e5,36079,1733534061701/ec1863dc21e5%2C36079%2C1733534061701.1733534063448 2024-12-07T01:14:23,482 DEBUG [RS:1;ec1863dc21e5:36079 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40865:40865),(127.0.0.1/127.0.0.1:39885:39885),(127.0.0.1/127.0.0.1:45341:45341)] 2024-12-07T01:14:23,484 INFO [RS:0;ec1863dc21e5:42915 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/WALs/ec1863dc21e5,42915,1733534061580/ec1863dc21e5%2C42915%2C1733534061580.1733534063448 2024-12-07T01:14:23,484 DEBUG [RS:0;ec1863dc21e5:42915 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39885:39885),(127.0.0.1/127.0.0.1:45341:45341),(127.0.0.1/127.0.0.1:40865:40865)] 2024-12-07T01:14:23,590 DEBUG [ec1863dc21e5:46141 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-12-07T01:14:23,599 DEBUG [ec1863dc21e5:46141 {}] balancer.BalancerClusterState(204): Hosts are {ec1863dc21e5=0} racks are {/default-rack=0} 2024-12-07T01:14:23,605 DEBUG [ec1863dc21e5:46141 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-07T01:14:23,605 DEBUG [ec1863dc21e5:46141 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-07T01:14:23,605 DEBUG [ec1863dc21e5:46141 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-12-07T01:14:23,605 DEBUG [ec1863dc21e5:46141 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-07T01:14:23,605 DEBUG [ec1863dc21e5:46141 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-07T01:14:23,605 DEBUG [ec1863dc21e5:46141 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-12-07T01:14:23,605 INFO [ec1863dc21e5:46141 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-07T01:14:23,605 INFO [ec1863dc21e5:46141 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-07T01:14:23,605 INFO [ec1863dc21e5:46141 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-12-07T01:14:23,605 DEBUG [ec1863dc21e5:46141 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-07T01:14:23,612 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=ec1863dc21e5,36079,1733534061701 2024-12-07T01:14:23,618 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ec1863dc21e5,36079,1733534061701, state=OPENING 2024-12-07T01:14:23,707 DEBUG [PEWorker-1 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-07T01:14:23,717 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46141-0x101ad5a899b0000, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T01:14:23,717 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40693-0x101ad5a899b0003, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T01:14:23,717 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36079-0x101ad5a899b0002, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T01:14:23,717 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42915-0x101ad5a899b0001, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T01:14:23,719 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T01:14:23,719 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T01:14:23,719 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T01:14:23,719 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T01:14:23,721 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-07T01:14:23,725 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=ec1863dc21e5,36079,1733534061701}] 2024-12-07T01:14:23,908 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-07T01:14:23,911 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60199, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-07T01:14:23,922 INFO [RS_OPEN_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-07T01:14:23,922 INFO [RS_OPEN_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-07T01:14:23,923 INFO [RS_OPEN_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-07T01:14:23,925 INFO [RS_OPEN_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ec1863dc21e5%2C36079%2C1733534061701.meta, suffix=.meta, logDir=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/WALs/ec1863dc21e5,36079,1733534061701, archiveDir=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/oldWALs, maxLogs=32 2024-12-07T01:14:23,938 DEBUG [RS_OPEN_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/WALs/ec1863dc21e5,36079,1733534061701/ec1863dc21e5%2C36079%2C1733534061701.meta.1733534063927.meta, exclude list is [], retry=0 2024-12-07T01:14:23,942 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37235,DS-b9cb305a-a2e1-439c-816b-c45d44c27920,DISK] 2024-12-07T01:14:23,942 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40127,DS-59048819-6295-483c-8edd-efcadeb66bf1,DISK] 2024-12-07T01:14:23,942 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43991,DS-f5ad7f4d-d62b-424d-bad1-dbf9e5694bea,DISK] 2024-12-07T01:14:23,945 INFO [RS_OPEN_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/WALs/ec1863dc21e5,36079,1733534061701/ec1863dc21e5%2C36079%2C1733534061701.meta.1733534063927.meta 2024-12-07T01:14:23,946 DEBUG [RS_OPEN_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39885:39885),(127.0.0.1/127.0.0.1:45341:45341),(127.0.0.1/127.0.0.1:40865:40865)] 2024-12-07T01:14:23,946 DEBUG [RS_OPEN_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-07T01:14:23,948 DEBUG [RS_OPEN_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-07T01:14:23,950 DEBUG [RS_OPEN_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-07T01:14:23,954 INFO [RS_OPEN_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-07T01:14:23,958 DEBUG [RS_OPEN_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-07T01:14:23,958 DEBUG [RS_OPEN_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T01:14:23,959 DEBUG [RS_OPEN_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-07T01:14:23,959 DEBUG [RS_OPEN_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-07T01:14:23,961 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-07T01:14:23,963 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-07T01:14:23,963 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:14:23,964 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T01:14:23,964 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-07T01:14:23,965 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-07T01:14:23,965 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:14:23,966 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T01:14:23,966 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-07T01:14:23,967 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-07T01:14:23,967 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:14:23,968 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T01:14:23,968 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-07T01:14:23,970 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-07T01:14:23,970 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:14:23,970 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T01:14:23,971 DEBUG [RS_OPEN_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-07T01:14:23,972 DEBUG [RS_OPEN_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/hbase/meta/1588230740 2024-12-07T01:14:23,974 DEBUG [RS_OPEN_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/hbase/meta/1588230740 2024-12-07T01:14:23,976 DEBUG [RS_OPEN_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-07T01:14:23,977 DEBUG [RS_OPEN_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-07T01:14:23,977 DEBUG [RS_OPEN_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-07T01:14:23,980 DEBUG [RS_OPEN_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-07T01:14:23,981 INFO [RS_OPEN_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67190857, jitterRate=0.0012217909097671509}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-07T01:14:23,982 DEBUG [RS_OPEN_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-07T01:14:23,983 DEBUG [RS_OPEN_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733534063959Writing region info on filesystem at 1733534063959Initializing all the Stores at 1733534063961 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733534063961Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733534063961Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534063961Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733534063961Cleaning up temporary data from old regions at 1733534063977 (+16 ms)Running coprocessor post-open hooks at 1733534063982 (+5 ms)Region opened successfully at 1733534063983 (+1 ms) 2024-12-07T01:14:23,989 INFO [RS_OPEN_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733534063898 2024-12-07T01:14:23,999 DEBUG [RS_OPEN_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-07T01:14:23,999 INFO [RS_OPEN_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-07T01:14:24,001 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=ec1863dc21e5,36079,1733534061701 2024-12-07T01:14:24,003 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ec1863dc21e5,36079,1733534061701, state=OPEN 2024-12-07T01:14:24,012 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40693-0x101ad5a899b0003, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T01:14:24,012 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42915-0x101ad5a899b0001, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T01:14:24,012 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36079-0x101ad5a899b0002, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T01:14:24,012 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T01:14:24,012 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T01:14:24,012 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T01:14:24,012 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46141-0x101ad5a899b0000, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T01:14:24,012 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T01:14:24,013 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=ec1863dc21e5,36079,1733534061701 2024-12-07T01:14:24,020 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-07T01:14:24,021 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ec1863dc21e5,36079,1733534061701 in 289 msec 2024-12-07T01:14:24,028 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-07T01:14:24,028 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 849 msec 2024-12-07T01:14:24,029 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-07T01:14:24,029 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-07T01:14:24,046 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T01:14:24,048 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ec1863dc21e5,36079,1733534061701, seqNum=-1] 2024-12-07T01:14:24,065 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T01:14:24,067 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35335, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T01:14:24,088 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.1240 sec 2024-12-07T01:14:24,088 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733534064088, completionTime=-1 2024-12-07T01:14:24,091 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-12-07T01:14:24,091 DEBUG [master/ec1863dc21e5:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-07T01:14:24,115 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=3 2024-12-07T01:14:24,115 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733534124115 2024-12-07T01:14:24,116 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733534184115 2024-12-07T01:14:24,116 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 24 msec 2024-12-07T01:14:24,118 DEBUG [master/ec1863dc21e5:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-12-07T01:14:24,126 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ec1863dc21e5,46141,1733534060840-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-07T01:14:24,127 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ec1863dc21e5,46141,1733534060840-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-07T01:14:24,127 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ec1863dc21e5,46141,1733534060840-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-07T01:14:24,129 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-ec1863dc21e5:46141, period=300000, unit=MILLISECONDS is enabled. 2024-12-07T01:14:24,129 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-07T01:14:24,130 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-07T01:14:24,135 DEBUG [master/ec1863dc21e5:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-07T01:14:24,158 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.249sec 2024-12-07T01:14:24,160 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-07T01:14:24,161 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-07T01:14:24,162 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-07T01:14:24,163 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-07T01:14:24,163 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-07T01:14:24,164 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ec1863dc21e5,46141,1733534060840-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-07T01:14:24,164 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ec1863dc21e5,46141,1733534060840-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-07T01:14:24,168 DEBUG [master/ec1863dc21e5:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-07T01:14:24,169 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-07T01:14:24,169 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ec1863dc21e5,46141,1733534060840-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-07T01:14:24,185 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9330f7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T01:14:24,190 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-07T01:14:24,190 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-07T01:14:24,194 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ec1863dc21e5,46141,-1 for getting cluster id 2024-12-07T01:14:24,197 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T01:14:24,205 DEBUG [HMasterForTest-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '68178764-2765-461b-a758-74a88455d04a' 2024-12-07T01:14:24,207 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T01:14:24,207 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "68178764-2765-461b-a758-74a88455d04a" 2024-12-07T01:14:24,208 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67351de1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T01:14:24,208 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ec1863dc21e5,46141,-1] 2024-12-07T01:14:24,210 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T01:14:24,212 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T01:14:24,213 INFO [HMasterForTest-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59846, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T01:14:24,215 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6982d266, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T01:14:24,216 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T01:14:24,223 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ec1863dc21e5,36079,1733534061701, seqNum=-1] 2024-12-07T01:14:24,223 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T01:14:24,226 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46080, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T01:14:24,263 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=ec1863dc21e5,46141,1733534060840 2024-12-07T01:14:24,267 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-07T01:14:24,273 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is ec1863dc21e5,46141,1733534060840 2024-12-07T01:14:24,279 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@485821e1 2024-12-07T01:14:24,280 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-07T01:14:24,283 INFO [HMasterForTest-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59850, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-07T01:14:24,292 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46141 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'test', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-07T01:14:24,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46141 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=test 2024-12-07T01:14:24,303 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_PRE_OPERATION 2024-12-07T01:14:24,305 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46141 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "test" procId is: 4 2024-12-07T01:14:24,306 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:14:24,309 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-07T01:14:24,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46141 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-07T01:14:24,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37235 is added to blk_1073741837_1013 (size=358) 2024-12-07T01:14:24,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741837_1013 (size=358) 2024-12-07T01:14:24,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43991 is added to blk_1073741837_1013 (size=358) 2024-12-07T01:14:24,337 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 6d8689bfa03ba7aed23b15e5ff5d23ac, NAME => 'test,ccc,1733534064285.6d8689bfa03ba7aed23b15e5ff5d23ac.', STARTKEY => 'ccc', ENDKEY => 'ddd'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323 2024-12-07T01:14:24,338 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 976bbe8d7afc3b1e02af11d1b515e82c, NAME => 'test,,1733534064285.976bbe8d7afc3b1e02af11d1b515e82c.', STARTKEY => '', ENDKEY => 'bbb'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323 2024-12-07T01:14:24,338 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 0980fd6969c12f09b843d772c626dfa5, NAME => 'test,bbb,1733534064285.0980fd6969c12f09b843d772c626dfa5.', STARTKEY => 'bbb', ENDKEY => 'ccc'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323 2024-12-07T01:14:24,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43991 is added to blk_1073741838_1014 (size=42) 2024-12-07T01:14:24,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741838_1014 (size=42) 2024-12-07T01:14:24,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37235 is added to blk_1073741838_1014 (size=42) 2024-12-07T01:14:24,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741839_1015 (size=45) 2024-12-07T01:14:24,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37235 is added to blk_1073741839_1015 (size=45) 2024-12-07T01:14:24,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43991 is added to blk_1073741839_1015 (size=45) 2024-12-07T01:14:24,352 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,,1733534064285.976bbe8d7afc3b1e02af11d1b515e82c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T01:14:24,353 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 976bbe8d7afc3b1e02af11d1b515e82c, disabling compactions & flushes 2024-12-07T01:14:24,353 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,,1733534064285.976bbe8d7afc3b1e02af11d1b515e82c. 2024-12-07T01:14:24,353 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,,1733534064285.976bbe8d7afc3b1e02af11d1b515e82c. 2024-12-07T01:14:24,353 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,,1733534064285.976bbe8d7afc3b1e02af11d1b515e82c. after waiting 0 ms 2024-12-07T01:14:24,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37235 is added to blk_1073741840_1016 (size=45) 2024-12-07T01:14:24,353 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,,1733534064285.976bbe8d7afc3b1e02af11d1b515e82c. 2024-12-07T01:14:24,353 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,,1733534064285.976bbe8d7afc3b1e02af11d1b515e82c. 2024-12-07T01:14:24,353 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 976bbe8d7afc3b1e02af11d1b515e82c: Waiting for close lock at 1733534064353Disabling compacts and flushes for region at 1733534064353Disabling writes for close at 1733534064353Writing region close event to WAL at 1733534064353Closed at 1733534064353 2024-12-07T01:14:24,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741840_1016 (size=45) 2024-12-07T01:14:24,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43991 is added to blk_1073741840_1016 (size=45) 2024-12-07T01:14:24,354 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => af843c9df2505d2eeee3c18062e9a2a6, NAME => 'test,ddd,1733534064285.af843c9df2505d2eeee3c18062e9a2a6.', STARTKEY => 'ddd', ENDKEY => 'eee'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323 2024-12-07T01:14:24,354 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,bbb,1733534064285.0980fd6969c12f09b843d772c626dfa5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T01:14:24,354 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 0980fd6969c12f09b843d772c626dfa5, disabling compactions & flushes 2024-12-07T01:14:24,354 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,bbb,1733534064285.0980fd6969c12f09b843d772c626dfa5. 2024-12-07T01:14:24,354 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,bbb,1733534064285.0980fd6969c12f09b843d772c626dfa5. 2024-12-07T01:14:24,354 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,bbb,1733534064285.0980fd6969c12f09b843d772c626dfa5. after waiting 0 ms 2024-12-07T01:14:24,354 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,bbb,1733534064285.0980fd6969c12f09b843d772c626dfa5. 2024-12-07T01:14:24,354 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,bbb,1733534064285.0980fd6969c12f09b843d772c626dfa5. 2024-12-07T01:14:24,354 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 0980fd6969c12f09b843d772c626dfa5: Waiting for close lock at 1733534064354Disabling compacts and flushes for region at 1733534064354Disabling writes for close at 1733534064354Writing region close event to WAL at 1733534064354Closed at 1733534064354 2024-12-07T01:14:24,355 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => c0a570532487ea0ebbc51cbec9ff7e37, NAME => 'test,eee,1733534064285.c0a570532487ea0ebbc51cbec9ff7e37.', STARTKEY => 'eee', ENDKEY => 'fff'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323 2024-12-07T01:14:24,355 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ccc,1733534064285.6d8689bfa03ba7aed23b15e5ff5d23ac.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T01:14:24,355 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 6d8689bfa03ba7aed23b15e5ff5d23ac, disabling compactions & flushes 2024-12-07T01:14:24,355 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ccc,1733534064285.6d8689bfa03ba7aed23b15e5ff5d23ac. 2024-12-07T01:14:24,355 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ccc,1733534064285.6d8689bfa03ba7aed23b15e5ff5d23ac. 2024-12-07T01:14:24,355 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ccc,1733534064285.6d8689bfa03ba7aed23b15e5ff5d23ac. after waiting 0 ms 2024-12-07T01:14:24,355 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ccc,1733534064285.6d8689bfa03ba7aed23b15e5ff5d23ac. 2024-12-07T01:14:24,355 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ccc,1733534064285.6d8689bfa03ba7aed23b15e5ff5d23ac. 2024-12-07T01:14:24,355 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 6d8689bfa03ba7aed23b15e5ff5d23ac: Waiting for close lock at 1733534064355Disabling compacts and flushes for region at 1733534064355Disabling writes for close at 1733534064355Writing region close event to WAL at 1733534064355Closed at 1733534064355 2024-12-07T01:14:24,356 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 37d131a6ac713be02dc73a29079210f5, NAME => 'test,fff,1733534064285.37d131a6ac713be02dc73a29079210f5.', STARTKEY => 'fff', ENDKEY => 'ggg'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323 2024-12-07T01:14:24,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43991 is added to blk_1073741841_1017 (size=45) 2024-12-07T01:14:24,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37235 is added to blk_1073741841_1017 (size=45) 2024-12-07T01:14:24,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741841_1017 (size=45) 2024-12-07T01:14:24,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741842_1018 (size=45) 2024-12-07T01:14:24,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43991 is added to blk_1073741842_1018 (size=45) 2024-12-07T01:14:24,367 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,ddd,1733534064285.af843c9df2505d2eeee3c18062e9a2a6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T01:14:24,367 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing af843c9df2505d2eeee3c18062e9a2a6, disabling compactions & flushes 2024-12-07T01:14:24,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37235 is added to blk_1073741842_1018 (size=45) 2024-12-07T01:14:24,367 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,ddd,1733534064285.af843c9df2505d2eeee3c18062e9a2a6. 2024-12-07T01:14:24,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741843_1019 (size=45) 2024-12-07T01:14:24,367 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ddd,1733534064285.af843c9df2505d2eeee3c18062e9a2a6. 2024-12-07T01:14:24,367 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,ddd,1733534064285.af843c9df2505d2eeee3c18062e9a2a6. after waiting 0 ms 2024-12-07T01:14:24,368 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,ddd,1733534064285.af843c9df2505d2eeee3c18062e9a2a6. 2024-12-07T01:14:24,368 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,ddd,1733534064285.af843c9df2505d2eeee3c18062e9a2a6. 2024-12-07T01:14:24,368 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for af843c9df2505d2eeee3c18062e9a2a6: Waiting for close lock at 1733534064367Disabling compacts and flushes for region at 1733534064367Disabling writes for close at 1733534064367Writing region close event to WAL at 1733534064368 (+1 ms)Closed at 1733534064368 2024-12-07T01:14:24,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43991 is added to blk_1073741843_1019 (size=45) 2024-12-07T01:14:24,368 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => c0f43810347933724c9e6319b11ae5cc, NAME => 'test,ggg,1733534064285.c0f43810347933724c9e6319b11ae5cc.', STARTKEY => 'ggg', ENDKEY => 'hhh'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323 2024-12-07T01:14:24,368 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,eee,1733534064285.c0a570532487ea0ebbc51cbec9ff7e37.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T01:14:24,368 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing c0a570532487ea0ebbc51cbec9ff7e37, disabling compactions & flushes 2024-12-07T01:14:24,368 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,eee,1733534064285.c0a570532487ea0ebbc51cbec9ff7e37. 2024-12-07T01:14:24,369 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,eee,1733534064285.c0a570532487ea0ebbc51cbec9ff7e37. 2024-12-07T01:14:24,369 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,eee,1733534064285.c0a570532487ea0ebbc51cbec9ff7e37. after waiting 0 ms 2024-12-07T01:14:24,369 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,eee,1733534064285.c0a570532487ea0ebbc51cbec9ff7e37. 2024-12-07T01:14:24,369 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,eee,1733534064285.c0a570532487ea0ebbc51cbec9ff7e37. 2024-12-07T01:14:24,369 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for c0a570532487ea0ebbc51cbec9ff7e37: Waiting for close lock at 1733534064368Disabling compacts and flushes for region at 1733534064368Disabling writes for close at 1733534064369 (+1 ms)Writing region close event to WAL at 1733534064369Closed at 1733534064369 2024-12-07T01:14:24,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37235 is added to blk_1073741843_1019 (size=45) 2024-12-07T01:14:24,370 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 9c426833151f173ef0d81d9c0ae11767, NAME => 'test,hhh,1733534064285.9c426833151f173ef0d81d9c0ae11767.', STARTKEY => 'hhh', ENDKEY => 'iii'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323 2024-12-07T01:14:24,370 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,fff,1733534064285.37d131a6ac713be02dc73a29079210f5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T01:14:24,370 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 37d131a6ac713be02dc73a29079210f5, disabling compactions & flushes 2024-12-07T01:14:24,370 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,fff,1733534064285.37d131a6ac713be02dc73a29079210f5. 2024-12-07T01:14:24,370 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,fff,1733534064285.37d131a6ac713be02dc73a29079210f5. 2024-12-07T01:14:24,370 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,fff,1733534064285.37d131a6ac713be02dc73a29079210f5. after waiting 0 ms 2024-12-07T01:14:24,370 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,fff,1733534064285.37d131a6ac713be02dc73a29079210f5. 2024-12-07T01:14:24,370 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,fff,1733534064285.37d131a6ac713be02dc73a29079210f5. 2024-12-07T01:14:24,370 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 37d131a6ac713be02dc73a29079210f5: Waiting for close lock at 1733534064370Disabling compacts and flushes for region at 1733534064370Disabling writes for close at 1733534064370Writing region close event to WAL at 1733534064370Closed at 1733534064370 2024-12-07T01:14:24,371 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 5d3306d206c48a1c7060e49d4fe00ad5, NAME => 'test,iii,1733534064285.5d3306d206c48a1c7060e49d4fe00ad5.', STARTKEY => 'iii', ENDKEY => 'jjj'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323 2024-12-07T01:14:24,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43991 is added to blk_1073741844_1020 (size=45) 2024-12-07T01:14:24,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37235 is added to blk_1073741844_1020 (size=45) 2024-12-07T01:14:24,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741844_1020 (size=45) 2024-12-07T01:14:24,380 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,ggg,1733534064285.c0f43810347933724c9e6319b11ae5cc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T01:14:24,380 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing c0f43810347933724c9e6319b11ae5cc, disabling compactions & flushes 2024-12-07T01:14:24,380 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,ggg,1733534064285.c0f43810347933724c9e6319b11ae5cc. 2024-12-07T01:14:24,381 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ggg,1733534064285.c0f43810347933724c9e6319b11ae5cc. 2024-12-07T01:14:24,381 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,ggg,1733534064285.c0f43810347933724c9e6319b11ae5cc. after waiting 0 ms 2024-12-07T01:14:24,381 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,ggg,1733534064285.c0f43810347933724c9e6319b11ae5cc. 2024-12-07T01:14:24,381 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,ggg,1733534064285.c0f43810347933724c9e6319b11ae5cc. 2024-12-07T01:14:24,381 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for c0f43810347933724c9e6319b11ae5cc: Waiting for close lock at 1733534064380Disabling compacts and flushes for region at 1733534064380Disabling writes for close at 1733534064381 (+1 ms)Writing region close event to WAL at 1733534064381Closed at 1733534064381 2024-12-07T01:14:24,381 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => aa3dca7b6c799eb77565fb0e20ef1c15, NAME => 'test,jjj,1733534064285.aa3dca7b6c799eb77565fb0e20ef1c15.', STARTKEY => 'jjj', ENDKEY => 'kkk'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323 2024-12-07T01:14:24,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37235 is added to blk_1073741845_1021 (size=45) 2024-12-07T01:14:24,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43991 is added to blk_1073741846_1022 (size=45) 2024-12-07T01:14:24,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741845_1021 (size=45) 2024-12-07T01:14:24,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43991 is added to blk_1073741845_1021 (size=45) 2024-12-07T01:14:24,384 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,hhh,1733534064285.9c426833151f173ef0d81d9c0ae11767.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T01:14:24,385 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 9c426833151f173ef0d81d9c0ae11767, disabling compactions & flushes 2024-12-07T01:14:24,385 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,hhh,1733534064285.9c426833151f173ef0d81d9c0ae11767. 2024-12-07T01:14:24,385 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,hhh,1733534064285.9c426833151f173ef0d81d9c0ae11767. 2024-12-07T01:14:24,385 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,hhh,1733534064285.9c426833151f173ef0d81d9c0ae11767. after waiting 0 ms 2024-12-07T01:14:24,385 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,hhh,1733534064285.9c426833151f173ef0d81d9c0ae11767. 2024-12-07T01:14:24,385 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,hhh,1733534064285.9c426833151f173ef0d81d9c0ae11767. 2024-12-07T01:14:24,385 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 9c426833151f173ef0d81d9c0ae11767: Waiting for close lock at 1733534064385Disabling compacts and flushes for region at 1733534064385Disabling writes for close at 1733534064385Writing region close event to WAL at 1733534064385Closed at 1733534064385 2024-12-07T01:14:24,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741846_1022 (size=45) 2024-12-07T01:14:24,386 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 75c927364905f2c6bffff16328d8aaf5, NAME => 'test,kkk,1733534064285.75c927364905f2c6bffff16328d8aaf5.', STARTKEY => 'kkk', ENDKEY => 'lll'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323 2024-12-07T01:14:24,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37235 is added to blk_1073741846_1022 (size=45) 2024-12-07T01:14:24,391 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,iii,1733534064285.5d3306d206c48a1c7060e49d4fe00ad5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T01:14:24,392 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 5d3306d206c48a1c7060e49d4fe00ad5, disabling compactions & flushes 2024-12-07T01:14:24,392 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,iii,1733534064285.5d3306d206c48a1c7060e49d4fe00ad5. 2024-12-07T01:14:24,392 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,iii,1733534064285.5d3306d206c48a1c7060e49d4fe00ad5. 2024-12-07T01:14:24,392 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,iii,1733534064285.5d3306d206c48a1c7060e49d4fe00ad5. after waiting 0 ms 2024-12-07T01:14:24,392 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,iii,1733534064285.5d3306d206c48a1c7060e49d4fe00ad5. 2024-12-07T01:14:24,392 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,iii,1733534064285.5d3306d206c48a1c7060e49d4fe00ad5. 2024-12-07T01:14:24,392 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 5d3306d206c48a1c7060e49d4fe00ad5: Waiting for close lock at 1733534064392Disabling compacts and flushes for region at 1733534064392Disabling writes for close at 1733534064392Writing region close event to WAL at 1733534064392Closed at 1733534064392 2024-12-07T01:14:24,393 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => d2b1eb41dda894f4f7b826a2826a9c7b, NAME => 'test,lll,1733534064285.d2b1eb41dda894f4f7b826a2826a9c7b.', STARTKEY => 'lll', ENDKEY => 'mmm'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323 2024-12-07T01:14:24,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37235 is added to blk_1073741847_1023 (size=45) 2024-12-07T01:14:24,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43991 is added to blk_1073741847_1023 (size=45) 2024-12-07T01:14:24,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741847_1023 (size=45) 2024-12-07T01:14:24,396 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,jjj,1733534064285.aa3dca7b6c799eb77565fb0e20ef1c15.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T01:14:24,397 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing aa3dca7b6c799eb77565fb0e20ef1c15, disabling compactions & flushes 2024-12-07T01:14:24,397 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,jjj,1733534064285.aa3dca7b6c799eb77565fb0e20ef1c15. 2024-12-07T01:14:24,397 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,jjj,1733534064285.aa3dca7b6c799eb77565fb0e20ef1c15. 2024-12-07T01:14:24,397 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,jjj,1733534064285.aa3dca7b6c799eb77565fb0e20ef1c15. after waiting 0 ms 2024-12-07T01:14:24,397 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,jjj,1733534064285.aa3dca7b6c799eb77565fb0e20ef1c15. 2024-12-07T01:14:24,397 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,jjj,1733534064285.aa3dca7b6c799eb77565fb0e20ef1c15. 2024-12-07T01:14:24,397 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for aa3dca7b6c799eb77565fb0e20ef1c15: Waiting for close lock at 1733534064397Disabling compacts and flushes for region at 1733534064397Disabling writes for close at 1733534064397Writing region close event to WAL at 1733534064397Closed at 1733534064397 2024-12-07T01:14:24,398 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 327d2bdcc20803fcfd1e61553f30068b, NAME => 'test,mmm,1733534064285.327d2bdcc20803fcfd1e61553f30068b.', STARTKEY => 'mmm', ENDKEY => 'nnn'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323 2024-12-07T01:14:24,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741848_1024 (size=45) 2024-12-07T01:14:24,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37235 is added to blk_1073741848_1024 (size=45) 2024-12-07T01:14:24,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43991 is added to blk_1073741848_1024 (size=45) 2024-12-07T01:14:24,406 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,kkk,1733534064285.75c927364905f2c6bffff16328d8aaf5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T01:14:24,406 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 75c927364905f2c6bffff16328d8aaf5, disabling compactions & flushes 2024-12-07T01:14:24,406 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,kkk,1733534064285.75c927364905f2c6bffff16328d8aaf5. 2024-12-07T01:14:24,406 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,kkk,1733534064285.75c927364905f2c6bffff16328d8aaf5. 2024-12-07T01:14:24,406 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,kkk,1733534064285.75c927364905f2c6bffff16328d8aaf5. after waiting 0 ms 2024-12-07T01:14:24,406 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,kkk,1733534064285.75c927364905f2c6bffff16328d8aaf5. 2024-12-07T01:14:24,406 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,kkk,1733534064285.75c927364905f2c6bffff16328d8aaf5. 2024-12-07T01:14:24,406 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 75c927364905f2c6bffff16328d8aaf5: Waiting for close lock at 1733534064406Disabling compacts and flushes for region at 1733534064406Disabling writes for close at 1733534064406Writing region close event to WAL at 1733534064406Closed at 1733534064406 2024-12-07T01:14:24,407 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 3fe1b6a546d269c366ae3f13263e1a2a, NAME => 'test,nnn,1733534064285.3fe1b6a546d269c366ae3f13263e1a2a.', STARTKEY => 'nnn', ENDKEY => 'ooo'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323 2024-12-07T01:14:24,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43991 is added to blk_1073741849_1025 (size=45) 2024-12-07T01:14:24,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741849_1025 (size=45) 2024-12-07T01:14:24,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37235 is added to blk_1073741849_1025 (size=45) 2024-12-07T01:14:24,414 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,lll,1733534064285.d2b1eb41dda894f4f7b826a2826a9c7b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T01:14:24,414 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing d2b1eb41dda894f4f7b826a2826a9c7b, disabling compactions & flushes 2024-12-07T01:14:24,414 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,lll,1733534064285.d2b1eb41dda894f4f7b826a2826a9c7b. 2024-12-07T01:14:24,414 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,lll,1733534064285.d2b1eb41dda894f4f7b826a2826a9c7b. 2024-12-07T01:14:24,414 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,lll,1733534064285.d2b1eb41dda894f4f7b826a2826a9c7b. after waiting 0 ms 2024-12-07T01:14:24,414 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,lll,1733534064285.d2b1eb41dda894f4f7b826a2826a9c7b. 2024-12-07T01:14:24,414 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,lll,1733534064285.d2b1eb41dda894f4f7b826a2826a9c7b. 2024-12-07T01:14:24,414 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for d2b1eb41dda894f4f7b826a2826a9c7b: Waiting for close lock at 1733534064414Disabling compacts and flushes for region at 1733534064414Disabling writes for close at 1733534064414Writing region close event to WAL at 1733534064414Closed at 1733534064414 2024-12-07T01:14:24,415 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 324bdaae20264b3880845c3082304e57, NAME => 'test,ooo,1733534064285.324bdaae20264b3880845c3082304e57.', STARTKEY => 'ooo', ENDKEY => 'ppp'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323 2024-12-07T01:14:24,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37235 is added to blk_1073741850_1026 (size=45) 2024-12-07T01:14:24,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43991 is added to blk_1073741850_1026 (size=45) 2024-12-07T01:14:24,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741850_1026 (size=45) 2024-12-07T01:14:24,420 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,mmm,1733534064285.327d2bdcc20803fcfd1e61553f30068b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T01:14:24,421 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 327d2bdcc20803fcfd1e61553f30068b, disabling compactions & flushes 2024-12-07T01:14:24,421 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,mmm,1733534064285.327d2bdcc20803fcfd1e61553f30068b. 2024-12-07T01:14:24,421 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,mmm,1733534064285.327d2bdcc20803fcfd1e61553f30068b. 2024-12-07T01:14:24,421 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,mmm,1733534064285.327d2bdcc20803fcfd1e61553f30068b. after waiting 0 ms 2024-12-07T01:14:24,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46141 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-07T01:14:24,421 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,mmm,1733534064285.327d2bdcc20803fcfd1e61553f30068b. 2024-12-07T01:14:24,421 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,mmm,1733534064285.327d2bdcc20803fcfd1e61553f30068b. 2024-12-07T01:14:24,421 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 327d2bdcc20803fcfd1e61553f30068b: Waiting for close lock at 1733534064420Disabling compacts and flushes for region at 1733534064420Disabling writes for close at 1733534064421 (+1 ms)Writing region close event to WAL at 1733534064421Closed at 1733534064421 2024-12-07T01:14:24,422 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => c66f402dd9e22edaa5996a3b7446808c, NAME => 'test,ppp,1733534064285.c66f402dd9e22edaa5996a3b7446808c.', STARTKEY => 'ppp', ENDKEY => 'qqq'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323 2024-12-07T01:14:24,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741851_1027 (size=45) 2024-12-07T01:14:24,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37235 is added to blk_1073741851_1027 (size=45) 2024-12-07T01:14:24,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43991 is added to blk_1073741851_1027 (size=45) 2024-12-07T01:14:24,426 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,nnn,1733534064285.3fe1b6a546d269c366ae3f13263e1a2a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T01:14:24,426 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 3fe1b6a546d269c366ae3f13263e1a2a, disabling compactions & flushes 2024-12-07T01:14:24,426 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,nnn,1733534064285.3fe1b6a546d269c366ae3f13263e1a2a. 2024-12-07T01:14:24,426 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,nnn,1733534064285.3fe1b6a546d269c366ae3f13263e1a2a. 2024-12-07T01:14:24,426 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,nnn,1733534064285.3fe1b6a546d269c366ae3f13263e1a2a. after waiting 0 ms 2024-12-07T01:14:24,426 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,nnn,1733534064285.3fe1b6a546d269c366ae3f13263e1a2a. 2024-12-07T01:14:24,426 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,nnn,1733534064285.3fe1b6a546d269c366ae3f13263e1a2a. 2024-12-07T01:14:24,426 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 3fe1b6a546d269c366ae3f13263e1a2a: Waiting for close lock at 1733534064426Disabling compacts and flushes for region at 1733534064426Disabling writes for close at 1733534064426Writing region close event to WAL at 1733534064426Closed at 1733534064426 2024-12-07T01:14:24,427 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 856730db89c49e65ccb40b5967ccf99d, NAME => 'test,qqq,1733534064285.856730db89c49e65ccb40b5967ccf99d.', STARTKEY => 'qqq', ENDKEY => 'rrr'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323 2024-12-07T01:14:24,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43991 is added to blk_1073741852_1028 (size=45) 2024-12-07T01:14:24,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741852_1028 (size=45) 2024-12-07T01:14:24,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37235 is added to blk_1073741852_1028 (size=45) 2024-12-07T01:14:24,432 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ooo,1733534064285.324bdaae20264b3880845c3082304e57.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T01:14:24,432 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 324bdaae20264b3880845c3082304e57, disabling compactions & flushes 2024-12-07T01:14:24,432 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ooo,1733534064285.324bdaae20264b3880845c3082304e57. 2024-12-07T01:14:24,432 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ooo,1733534064285.324bdaae20264b3880845c3082304e57. 2024-12-07T01:14:24,432 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ooo,1733534064285.324bdaae20264b3880845c3082304e57. after waiting 0 ms 2024-12-07T01:14:24,432 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ooo,1733534064285.324bdaae20264b3880845c3082304e57. 2024-12-07T01:14:24,432 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ooo,1733534064285.324bdaae20264b3880845c3082304e57. 2024-12-07T01:14:24,432 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 324bdaae20264b3880845c3082304e57: Waiting for close lock at 1733534064432Disabling compacts and flushes for region at 1733534064432Disabling writes for close at 1733534064432Writing region close event to WAL at 1733534064432Closed at 1733534064432 2024-12-07T01:14:24,433 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => fa4e20ed49e59ee4cf63b6d84515872f, NAME => 'test,rrr,1733534064285.fa4e20ed49e59ee4cf63b6d84515872f.', STARTKEY => 'rrr', ENDKEY => 'sss'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323 2024-12-07T01:14:24,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741853_1029 (size=45) 2024-12-07T01:14:24,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43991 is added to blk_1073741853_1029 (size=45) 2024-12-07T01:14:24,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37235 is added to blk_1073741853_1029 (size=45) 2024-12-07T01:14:24,441 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,ppp,1733534064285.c66f402dd9e22edaa5996a3b7446808c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T01:14:24,441 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing c66f402dd9e22edaa5996a3b7446808c, disabling compactions & flushes 2024-12-07T01:14:24,441 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,ppp,1733534064285.c66f402dd9e22edaa5996a3b7446808c. 2024-12-07T01:14:24,441 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ppp,1733534064285.c66f402dd9e22edaa5996a3b7446808c. 2024-12-07T01:14:24,441 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,ppp,1733534064285.c66f402dd9e22edaa5996a3b7446808c. after waiting 0 ms 2024-12-07T01:14:24,441 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,ppp,1733534064285.c66f402dd9e22edaa5996a3b7446808c. 2024-12-07T01:14:24,441 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,ppp,1733534064285.c66f402dd9e22edaa5996a3b7446808c. 2024-12-07T01:14:24,441 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for c66f402dd9e22edaa5996a3b7446808c: Waiting for close lock at 1733534064441Disabling compacts and flushes for region at 1733534064441Disabling writes for close at 1733534064441Writing region close event to WAL at 1733534064441Closed at 1733534064441 2024-12-07T01:14:24,442 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 1da36e78b320d37341113b4185c91b26, NAME => 'test,sss,1733534064285.1da36e78b320d37341113b4185c91b26.', STARTKEY => 'sss', ENDKEY => 'ttt'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323 2024-12-07T01:14:24,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37235 is added to blk_1073741854_1030 (size=45) 2024-12-07T01:14:24,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741854_1030 (size=45) 2024-12-07T01:14:24,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43991 is added to blk_1073741854_1030 (size=45) 2024-12-07T01:14:24,445 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,qqq,1733534064285.856730db89c49e65ccb40b5967ccf99d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T01:14:24,446 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 856730db89c49e65ccb40b5967ccf99d, disabling compactions & flushes 2024-12-07T01:14:24,446 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,qqq,1733534064285.856730db89c49e65ccb40b5967ccf99d. 2024-12-07T01:14:24,446 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,qqq,1733534064285.856730db89c49e65ccb40b5967ccf99d. 2024-12-07T01:14:24,446 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,qqq,1733534064285.856730db89c49e65ccb40b5967ccf99d. after waiting 0 ms 2024-12-07T01:14:24,446 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,qqq,1733534064285.856730db89c49e65ccb40b5967ccf99d. 2024-12-07T01:14:24,446 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,qqq,1733534064285.856730db89c49e65ccb40b5967ccf99d. 2024-12-07T01:14:24,446 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 856730db89c49e65ccb40b5967ccf99d: Waiting for close lock at 1733534064446Disabling compacts and flushes for region at 1733534064446Disabling writes for close at 1733534064446Writing region close event to WAL at 1733534064446Closed at 1733534064446 2024-12-07T01:14:24,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37235 is added to blk_1073741855_1031 (size=45) 2024-12-07T01:14:24,447 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 3f148a9f8c5bea63f4744afb36553ef7, NAME => 'test,ttt,1733534064285.3f148a9f8c5bea63f4744afb36553ef7.', STARTKEY => 'ttt', ENDKEY => 'uuu'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323 2024-12-07T01:14:24,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741855_1031 (size=45) 2024-12-07T01:14:24,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43991 is added to blk_1073741855_1031 (size=45) 2024-12-07T01:14:24,450 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,rrr,1733534064285.fa4e20ed49e59ee4cf63b6d84515872f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T01:14:24,450 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing fa4e20ed49e59ee4cf63b6d84515872f, disabling compactions & flushes 2024-12-07T01:14:24,450 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,rrr,1733534064285.fa4e20ed49e59ee4cf63b6d84515872f. 2024-12-07T01:14:24,450 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,rrr,1733534064285.fa4e20ed49e59ee4cf63b6d84515872f. 2024-12-07T01:14:24,450 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,rrr,1733534064285.fa4e20ed49e59ee4cf63b6d84515872f. after waiting 0 ms 2024-12-07T01:14:24,450 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,rrr,1733534064285.fa4e20ed49e59ee4cf63b6d84515872f. 2024-12-07T01:14:24,450 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,rrr,1733534064285.fa4e20ed49e59ee4cf63b6d84515872f. 2024-12-07T01:14:24,450 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for fa4e20ed49e59ee4cf63b6d84515872f: Waiting for close lock at 1733534064450Disabling compacts and flushes for region at 1733534064450Disabling writes for close at 1733534064450Writing region close event to WAL at 1733534064450Closed at 1733534064450 2024-12-07T01:14:24,451 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 80bacdcc81a44a51d622e9d873892ea3, NAME => 'test,uuu,1733534064285.80bacdcc81a44a51d622e9d873892ea3.', STARTKEY => 'uuu', ENDKEY => 'vvv'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323 2024-12-07T01:14:24,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43991 is added to blk_1073741856_1032 (size=45) 2024-12-07T01:14:24,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37235 is added to blk_1073741856_1032 (size=45) 2024-12-07T01:14:24,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741856_1032 (size=45) 2024-12-07T01:14:24,456 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,sss,1733534064285.1da36e78b320d37341113b4185c91b26.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T01:14:24,456 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 1da36e78b320d37341113b4185c91b26, disabling compactions & flushes 2024-12-07T01:14:24,456 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,sss,1733534064285.1da36e78b320d37341113b4185c91b26. 2024-12-07T01:14:24,456 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,sss,1733534064285.1da36e78b320d37341113b4185c91b26. 2024-12-07T01:14:24,456 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,sss,1733534064285.1da36e78b320d37341113b4185c91b26. after waiting 0 ms 2024-12-07T01:14:24,456 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,sss,1733534064285.1da36e78b320d37341113b4185c91b26. 2024-12-07T01:14:24,456 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,sss,1733534064285.1da36e78b320d37341113b4185c91b26. 2024-12-07T01:14:24,456 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 1da36e78b320d37341113b4185c91b26: Waiting for close lock at 1733534064456Disabling compacts and flushes for region at 1733534064456Disabling writes for close at 1733534064456Writing region close event to WAL at 1733534064456Closed at 1733534064456 2024-12-07T01:14:24,457 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 76cd5c1764feda4c992319b218572cef, NAME => 'test,vvv,1733534064285.76cd5c1764feda4c992319b218572cef.', STARTKEY => 'vvv', ENDKEY => 'www'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323 2024-12-07T01:14:24,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741857_1033 (size=45) 2024-12-07T01:14:24,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43991 is added to blk_1073741857_1033 (size=45) 2024-12-07T01:14:24,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37235 is added to blk_1073741857_1033 (size=45) 2024-12-07T01:14:24,461 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,ttt,1733534064285.3f148a9f8c5bea63f4744afb36553ef7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T01:14:24,461 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 3f148a9f8c5bea63f4744afb36553ef7, disabling compactions & flushes 2024-12-07T01:14:24,461 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,ttt,1733534064285.3f148a9f8c5bea63f4744afb36553ef7. 2024-12-07T01:14:24,461 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ttt,1733534064285.3f148a9f8c5bea63f4744afb36553ef7. 2024-12-07T01:14:24,461 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,ttt,1733534064285.3f148a9f8c5bea63f4744afb36553ef7. after waiting 0 ms 2024-12-07T01:14:24,461 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,ttt,1733534064285.3f148a9f8c5bea63f4744afb36553ef7. 2024-12-07T01:14:24,461 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,ttt,1733534064285.3f148a9f8c5bea63f4744afb36553ef7. 2024-12-07T01:14:24,461 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 3f148a9f8c5bea63f4744afb36553ef7: Waiting for close lock at 1733534064461Disabling compacts and flushes for region at 1733534064461Disabling writes for close at 1733534064461Writing region close event to WAL at 1733534064461Closed at 1733534064461 2024-12-07T01:14:24,462 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 8cd2040dc039667d68bc1a24f5f2322f, NAME => 'test,www,1733534064285.8cd2040dc039667d68bc1a24f5f2322f.', STARTKEY => 'www', ENDKEY => 'xxx'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323 2024-12-07T01:14:24,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43991 is added to blk_1073741858_1034 (size=45) 2024-12-07T01:14:24,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741858_1034 (size=45) 2024-12-07T01:14:24,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37235 is added to blk_1073741858_1034 (size=45) 2024-12-07T01:14:24,469 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,uuu,1733534064285.80bacdcc81a44a51d622e9d873892ea3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T01:14:24,469 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 80bacdcc81a44a51d622e9d873892ea3, disabling compactions & flushes 2024-12-07T01:14:24,469 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,uuu,1733534064285.80bacdcc81a44a51d622e9d873892ea3. 2024-12-07T01:14:24,469 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,uuu,1733534064285.80bacdcc81a44a51d622e9d873892ea3. 2024-12-07T01:14:24,469 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,uuu,1733534064285.80bacdcc81a44a51d622e9d873892ea3. after waiting 0 ms 2024-12-07T01:14:24,469 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,uuu,1733534064285.80bacdcc81a44a51d622e9d873892ea3. 2024-12-07T01:14:24,469 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,uuu,1733534064285.80bacdcc81a44a51d622e9d873892ea3. 2024-12-07T01:14:24,469 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 80bacdcc81a44a51d622e9d873892ea3: Waiting for close lock at 1733534064469Disabling compacts and flushes for region at 1733534064469Disabling writes for close at 1733534064469Writing region close event to WAL at 1733534064469Closed at 1733534064469 2024-12-07T01:14:24,470 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 1cf1ce3ee0822ff957b8ffa13f56df7d, NAME => 'test,xxx,1733534064285.1cf1ce3ee0822ff957b8ffa13f56df7d.', STARTKEY => 'xxx', ENDKEY => 'yyy'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323 2024-12-07T01:14:24,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741859_1035 (size=45) 2024-12-07T01:14:24,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37235 is added to blk_1073741859_1035 (size=45) 2024-12-07T01:14:24,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43991 is added to blk_1073741859_1035 (size=45) 2024-12-07T01:14:24,472 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,vvv,1733534064285.76cd5c1764feda4c992319b218572cef.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T01:14:24,473 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 76cd5c1764feda4c992319b218572cef, disabling compactions & flushes 2024-12-07T01:14:24,473 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,vvv,1733534064285.76cd5c1764feda4c992319b218572cef. 2024-12-07T01:14:24,473 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,vvv,1733534064285.76cd5c1764feda4c992319b218572cef. 2024-12-07T01:14:24,473 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,vvv,1733534064285.76cd5c1764feda4c992319b218572cef. after waiting 0 ms 2024-12-07T01:14:24,473 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,vvv,1733534064285.76cd5c1764feda4c992319b218572cef. 2024-12-07T01:14:24,473 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,vvv,1733534064285.76cd5c1764feda4c992319b218572cef. 2024-12-07T01:14:24,473 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 76cd5c1764feda4c992319b218572cef: Waiting for close lock at 1733534064473Disabling compacts and flushes for region at 1733534064473Disabling writes for close at 1733534064473Writing region close event to WAL at 1733534064473Closed at 1733534064473 2024-12-07T01:14:24,474 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => b59f40eb016096571b13ae445f262449, NAME => 'test,yyy,1733534064285.b59f40eb016096571b13ae445f262449.', STARTKEY => 'yyy', ENDKEY => 'zzz'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323 2024-12-07T01:14:24,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43991 is added to blk_1073741860_1036 (size=45) 2024-12-07T01:14:24,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37235 is added to blk_1073741860_1036 (size=45) 2024-12-07T01:14:24,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741860_1036 (size=45) 2024-12-07T01:14:24,479 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,www,1733534064285.8cd2040dc039667d68bc1a24f5f2322f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T01:14:24,479 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 8cd2040dc039667d68bc1a24f5f2322f, disabling compactions & flushes 2024-12-07T01:14:24,479 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,www,1733534064285.8cd2040dc039667d68bc1a24f5f2322f. 2024-12-07T01:14:24,479 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,www,1733534064285.8cd2040dc039667d68bc1a24f5f2322f. 2024-12-07T01:14:24,479 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,www,1733534064285.8cd2040dc039667d68bc1a24f5f2322f. after waiting 0 ms 2024-12-07T01:14:24,480 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,www,1733534064285.8cd2040dc039667d68bc1a24f5f2322f. 2024-12-07T01:14:24,480 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,www,1733534064285.8cd2040dc039667d68bc1a24f5f2322f. 2024-12-07T01:14:24,480 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 8cd2040dc039667d68bc1a24f5f2322f: Waiting for close lock at 1733534064479Disabling compacts and flushes for region at 1733534064479Disabling writes for close at 1733534064479Writing region close event to WAL at 1733534064480 (+1 ms)Closed at 1733534064480 2024-12-07T01:14:24,481 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 73ec7ce1c93542e31d99b7b4474a9750, NAME => 'test,zzz,1733534064285.73ec7ce1c93542e31d99b7b4474a9750.', STARTKEY => 'zzz', ENDKEY => ''}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323 2024-12-07T01:14:24,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37235 is added to blk_1073741861_1037 (size=45) 2024-12-07T01:14:24,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43991 is added to blk_1073741861_1037 (size=45) 2024-12-07T01:14:24,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741861_1037 (size=45) 2024-12-07T01:14:24,484 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,xxx,1733534064285.1cf1ce3ee0822ff957b8ffa13f56df7d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T01:14:24,484 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 1cf1ce3ee0822ff957b8ffa13f56df7d, disabling compactions & flushes 2024-12-07T01:14:24,484 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,xxx,1733534064285.1cf1ce3ee0822ff957b8ffa13f56df7d. 2024-12-07T01:14:24,485 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,xxx,1733534064285.1cf1ce3ee0822ff957b8ffa13f56df7d. 2024-12-07T01:14:24,485 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,xxx,1733534064285.1cf1ce3ee0822ff957b8ffa13f56df7d. after waiting 0 ms 2024-12-07T01:14:24,485 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,xxx,1733534064285.1cf1ce3ee0822ff957b8ffa13f56df7d. 2024-12-07T01:14:24,485 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,xxx,1733534064285.1cf1ce3ee0822ff957b8ffa13f56df7d. 2024-12-07T01:14:24,485 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 1cf1ce3ee0822ff957b8ffa13f56df7d: Waiting for close lock at 1733534064484Disabling compacts and flushes for region at 1733534064484Disabling writes for close at 1733534064485 (+1 ms)Writing region close event to WAL at 1733534064485Closed at 1733534064485 2024-12-07T01:14:24,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741862_1038 (size=45) 2024-12-07T01:14:24,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37235 is added to blk_1073741862_1038 (size=45) 2024-12-07T01:14:24,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43991 is added to blk_1073741862_1038 (size=45) 2024-12-07T01:14:24,489 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,yyy,1733534064285.b59f40eb016096571b13ae445f262449.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T01:14:24,489 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing b59f40eb016096571b13ae445f262449, disabling compactions & flushes 2024-12-07T01:14:24,489 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,yyy,1733534064285.b59f40eb016096571b13ae445f262449. 2024-12-07T01:14:24,489 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,yyy,1733534064285.b59f40eb016096571b13ae445f262449. 2024-12-07T01:14:24,489 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,yyy,1733534064285.b59f40eb016096571b13ae445f262449. after waiting 0 ms 2024-12-07T01:14:24,489 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,yyy,1733534064285.b59f40eb016096571b13ae445f262449. 2024-12-07T01:14:24,490 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,yyy,1733534064285.b59f40eb016096571b13ae445f262449. 2024-12-07T01:14:24,490 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for b59f40eb016096571b13ae445f262449: Waiting for close lock at 1733534064489Disabling compacts and flushes for region at 1733534064489Disabling writes for close at 1733534064489Writing region close event to WAL at 1733534064490 (+1 ms)Closed at 1733534064490 2024-12-07T01:14:24,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43991 is added to blk_1073741863_1039 (size=42) 2024-12-07T01:14:24,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37235 is added to blk_1073741863_1039 (size=42) 2024-12-07T01:14:24,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741863_1039 (size=42) 2024-12-07T01:14:24,494 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,zzz,1733534064285.73ec7ce1c93542e31d99b7b4474a9750.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T01:14:24,494 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 73ec7ce1c93542e31d99b7b4474a9750, disabling compactions & flushes 2024-12-07T01:14:24,494 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,zzz,1733534064285.73ec7ce1c93542e31d99b7b4474a9750. 2024-12-07T01:14:24,494 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,zzz,1733534064285.73ec7ce1c93542e31d99b7b4474a9750. 2024-12-07T01:14:24,494 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,zzz,1733534064285.73ec7ce1c93542e31d99b7b4474a9750. after waiting 0 ms 2024-12-07T01:14:24,494 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,zzz,1733534064285.73ec7ce1c93542e31d99b7b4474a9750. 2024-12-07T01:14:24,494 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,zzz,1733534064285.73ec7ce1c93542e31d99b7b4474a9750. 2024-12-07T01:14:24,494 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 73ec7ce1c93542e31d99b7b4474a9750: Waiting for close lock at 1733534064494Disabling compacts and flushes for region at 1733534064494Disabling writes for close at 1733534064494Writing region close event to WAL at 1733534064494Closed at 1733534064494 2024-12-07T01:14:24,496 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_ADD_TO_META 2024-12-07T01:14:24,501 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,,1733534064285.976bbe8d7afc3b1e02af11d1b515e82c.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1733534064496"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733534064496"}]},"ts":"1733534064496"} 2024-12-07T01:14:24,501 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,bbb,1733534064285.0980fd6969c12f09b843d772c626dfa5.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733534064496"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733534064496"}]},"ts":"1733534064496"} 2024-12-07T01:14:24,501 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ccc,1733534064285.6d8689bfa03ba7aed23b15e5ff5d23ac.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733534064496"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733534064496"}]},"ts":"1733534064496"} 2024-12-07T01:14:24,501 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ddd,1733534064285.af843c9df2505d2eeee3c18062e9a2a6.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733534064496"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733534064496"}]},"ts":"1733534064496"} 2024-12-07T01:14:24,501 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,eee,1733534064285.c0a570532487ea0ebbc51cbec9ff7e37.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733534064496"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733534064496"}]},"ts":"1733534064496"} 2024-12-07T01:14:24,501 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,fff,1733534064285.37d131a6ac713be02dc73a29079210f5.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733534064496"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733534064496"}]},"ts":"1733534064496"} 2024-12-07T01:14:24,502 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ggg,1733534064285.c0f43810347933724c9e6319b11ae5cc.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733534064496"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733534064496"}]},"ts":"1733534064496"} 2024-12-07T01:14:24,502 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,hhh,1733534064285.9c426833151f173ef0d81d9c0ae11767.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733534064496"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733534064496"}]},"ts":"1733534064496"} 2024-12-07T01:14:24,502 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,iii,1733534064285.5d3306d206c48a1c7060e49d4fe00ad5.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733534064496"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733534064496"}]},"ts":"1733534064496"} 2024-12-07T01:14:24,502 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,jjj,1733534064285.aa3dca7b6c799eb77565fb0e20ef1c15.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733534064496"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733534064496"}]},"ts":"1733534064496"} 2024-12-07T01:14:24,502 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,kkk,1733534064285.75c927364905f2c6bffff16328d8aaf5.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733534064496"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733534064496"}]},"ts":"1733534064496"} 2024-12-07T01:14:24,502 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,lll,1733534064285.d2b1eb41dda894f4f7b826a2826a9c7b.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733534064496"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733534064496"}]},"ts":"1733534064496"} 2024-12-07T01:14:24,502 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,mmm,1733534064285.327d2bdcc20803fcfd1e61553f30068b.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733534064496"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733534064496"}]},"ts":"1733534064496"} 2024-12-07T01:14:24,502 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,nnn,1733534064285.3fe1b6a546d269c366ae3f13263e1a2a.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733534064496"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733534064496"}]},"ts":"1733534064496"} 2024-12-07T01:14:24,502 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ooo,1733534064285.324bdaae20264b3880845c3082304e57.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733534064496"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733534064496"}]},"ts":"1733534064496"} 2024-12-07T01:14:24,503 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ppp,1733534064285.c66f402dd9e22edaa5996a3b7446808c.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733534064496"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733534064496"}]},"ts":"1733534064496"} 2024-12-07T01:14:24,503 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,qqq,1733534064285.856730db89c49e65ccb40b5967ccf99d.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733534064496"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733534064496"}]},"ts":"1733534064496"} 2024-12-07T01:14:24,503 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,rrr,1733534064285.fa4e20ed49e59ee4cf63b6d84515872f.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733534064496"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733534064496"}]},"ts":"1733534064496"} 2024-12-07T01:14:24,503 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,sss,1733534064285.1da36e78b320d37341113b4185c91b26.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733534064496"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733534064496"}]},"ts":"1733534064496"} 2024-12-07T01:14:24,503 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ttt,1733534064285.3f148a9f8c5bea63f4744afb36553ef7.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733534064496"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733534064496"}]},"ts":"1733534064496"} 2024-12-07T01:14:24,503 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,uuu,1733534064285.80bacdcc81a44a51d622e9d873892ea3.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733534064496"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733534064496"}]},"ts":"1733534064496"} 2024-12-07T01:14:24,503 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,vvv,1733534064285.76cd5c1764feda4c992319b218572cef.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733534064496"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733534064496"}]},"ts":"1733534064496"} 2024-12-07T01:14:24,503 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,www,1733534064285.8cd2040dc039667d68bc1a24f5f2322f.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733534064496"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733534064496"}]},"ts":"1733534064496"} 2024-12-07T01:14:24,503 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,xxx,1733534064285.1cf1ce3ee0822ff957b8ffa13f56df7d.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733534064496"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733534064496"}]},"ts":"1733534064496"} 2024-12-07T01:14:24,503 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,yyy,1733534064285.b59f40eb016096571b13ae445f262449.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733534064496"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733534064496"}]},"ts":"1733534064496"} 2024-12-07T01:14:24,504 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,zzz,1733534064285.73ec7ce1c93542e31d99b7b4474a9750.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1733534064496"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733534064496"}]},"ts":"1733534064496"} 2024-12-07T01:14:24,542 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 26 regions to meta. 2024-12-07T01:14:24,544 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-07T01:14:24,547 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733534064544"}]},"ts":"1733534064544"} 2024-12-07T01:14:24,551 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=test, state=ENABLING in hbase:meta 2024-12-07T01:14:24,552 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(204): Hosts are {ec1863dc21e5=0} racks are {/default-rack=0} 2024-12-07T01:14:24,567 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-07T01:14:24,567 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-07T01:14:24,567 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-12-07T01:14:24,567 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-07T01:14:24,567 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-07T01:14:24,567 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-12-07T01:14:24,567 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-07T01:14:24,567 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-07T01:14:24,567 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-12-07T01:14:24,567 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-07T01:14:24,569 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=976bbe8d7afc3b1e02af11d1b515e82c, ASSIGN}, {pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=0980fd6969c12f09b843d772c626dfa5, ASSIGN}, {pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=6d8689bfa03ba7aed23b15e5ff5d23ac, ASSIGN}, {pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=af843c9df2505d2eeee3c18062e9a2a6, ASSIGN}, {pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c0a570532487ea0ebbc51cbec9ff7e37, ASSIGN}, {pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=37d131a6ac713be02dc73a29079210f5, ASSIGN}, {pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c0f43810347933724c9e6319b11ae5cc, ASSIGN}, {pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=9c426833151f173ef0d81d9c0ae11767, ASSIGN}, {pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=5d3306d206c48a1c7060e49d4fe00ad5, ASSIGN}, {pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=aa3dca7b6c799eb77565fb0e20ef1c15, ASSIGN}, {pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=75c927364905f2c6bffff16328d8aaf5, ASSIGN}, {pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d2b1eb41dda894f4f7b826a2826a9c7b, ASSIGN}, {pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=327d2bdcc20803fcfd1e61553f30068b, ASSIGN}, {pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=3fe1b6a546d269c366ae3f13263e1a2a, ASSIGN}, {pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=324bdaae20264b3880845c3082304e57, ASSIGN}, {pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c66f402dd9e22edaa5996a3b7446808c, ASSIGN}, {pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=856730db89c49e65ccb40b5967ccf99d, ASSIGN}, {pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=fa4e20ed49e59ee4cf63b6d84515872f, ASSIGN}, {pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=1da36e78b320d37341113b4185c91b26, ASSIGN}, {pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=3f148a9f8c5bea63f4744afb36553ef7, ASSIGN}, {pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=80bacdcc81a44a51d622e9d873892ea3, ASSIGN}, {pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=76cd5c1764feda4c992319b218572cef, ASSIGN}, {pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8cd2040dc039667d68bc1a24f5f2322f, ASSIGN}, {pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=1cf1ce3ee0822ff957b8ffa13f56df7d, ASSIGN}, {pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=b59f40eb016096571b13ae445f262449, ASSIGN}, {pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=73ec7ce1c93542e31d99b7b4474a9750, ASSIGN}] 2024-12-07T01:14:24,577 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=73ec7ce1c93542e31d99b7b4474a9750, ASSIGN 2024-12-07T01:14:24,579 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=73ec7ce1c93542e31d99b7b4474a9750, ASSIGN; state=OFFLINE, location=ec1863dc21e5,42915,1733534061580; forceNewPlan=false, retain=false 2024-12-07T01:14:24,580 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=b59f40eb016096571b13ae445f262449, ASSIGN 2024-12-07T01:14:24,581 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=b59f40eb016096571b13ae445f262449, ASSIGN; state=OFFLINE, location=ec1863dc21e5,40693,1733534061744; forceNewPlan=false, retain=false 2024-12-07T01:14:24,583 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=1cf1ce3ee0822ff957b8ffa13f56df7d, ASSIGN 2024-12-07T01:14:24,584 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=1cf1ce3ee0822ff957b8ffa13f56df7d, ASSIGN; state=OFFLINE, location=ec1863dc21e5,36079,1733534061701; forceNewPlan=false, retain=false 2024-12-07T01:14:24,586 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8cd2040dc039667d68bc1a24f5f2322f, ASSIGN 2024-12-07T01:14:24,587 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=8cd2040dc039667d68bc1a24f5f2322f, ASSIGN; state=OFFLINE, location=ec1863dc21e5,42915,1733534061580; forceNewPlan=false, retain=false 2024-12-07T01:14:24,589 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=76cd5c1764feda4c992319b218572cef, ASSIGN 2024-12-07T01:14:24,590 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=76cd5c1764feda4c992319b218572cef, ASSIGN; state=OFFLINE, location=ec1863dc21e5,40693,1733534061744; forceNewPlan=false, retain=false 2024-12-07T01:14:24,592 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=80bacdcc81a44a51d622e9d873892ea3, ASSIGN 2024-12-07T01:14:24,593 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=80bacdcc81a44a51d622e9d873892ea3, ASSIGN; state=OFFLINE, location=ec1863dc21e5,36079,1733534061701; forceNewPlan=false, retain=false 2024-12-07T01:14:24,595 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=3f148a9f8c5bea63f4744afb36553ef7, ASSIGN 2024-12-07T01:14:24,596 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=3f148a9f8c5bea63f4744afb36553ef7, ASSIGN; state=OFFLINE, location=ec1863dc21e5,42915,1733534061580; forceNewPlan=false, retain=false 2024-12-07T01:14:24,598 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=1da36e78b320d37341113b4185c91b26, ASSIGN 2024-12-07T01:14:24,599 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=1da36e78b320d37341113b4185c91b26, ASSIGN; state=OFFLINE, location=ec1863dc21e5,40693,1733534061744; forceNewPlan=false, retain=false 2024-12-07T01:14:24,601 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=fa4e20ed49e59ee4cf63b6d84515872f, ASSIGN 2024-12-07T01:14:24,603 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=fa4e20ed49e59ee4cf63b6d84515872f, ASSIGN; state=OFFLINE, location=ec1863dc21e5,36079,1733534061701; forceNewPlan=false, retain=false 2024-12-07T01:14:24,605 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=856730db89c49e65ccb40b5967ccf99d, ASSIGN 2024-12-07T01:14:24,607 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=856730db89c49e65ccb40b5967ccf99d, ASSIGN; state=OFFLINE, location=ec1863dc21e5,42915,1733534061580; forceNewPlan=false, retain=false 2024-12-07T01:14:24,608 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c66f402dd9e22edaa5996a3b7446808c, ASSIGN 2024-12-07T01:14:24,610 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=c66f402dd9e22edaa5996a3b7446808c, ASSIGN; state=OFFLINE, location=ec1863dc21e5,40693,1733534061744; forceNewPlan=false, retain=false 2024-12-07T01:14:24,611 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=324bdaae20264b3880845c3082304e57, ASSIGN 2024-12-07T01:14:24,613 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=324bdaae20264b3880845c3082304e57, ASSIGN; state=OFFLINE, location=ec1863dc21e5,36079,1733534061701; forceNewPlan=false, retain=false 2024-12-07T01:14:24,614 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=3fe1b6a546d269c366ae3f13263e1a2a, ASSIGN 2024-12-07T01:14:24,615 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=3fe1b6a546d269c366ae3f13263e1a2a, ASSIGN; state=OFFLINE, location=ec1863dc21e5,42915,1733534061580; forceNewPlan=false, retain=false 2024-12-07T01:14:24,616 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=327d2bdcc20803fcfd1e61553f30068b, ASSIGN 2024-12-07T01:14:24,618 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=327d2bdcc20803fcfd1e61553f30068b, ASSIGN; state=OFFLINE, location=ec1863dc21e5,40693,1733534061744; forceNewPlan=false, retain=false 2024-12-07T01:14:24,619 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d2b1eb41dda894f4f7b826a2826a9c7b, ASSIGN 2024-12-07T01:14:24,620 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=d2b1eb41dda894f4f7b826a2826a9c7b, ASSIGN; state=OFFLINE, location=ec1863dc21e5,36079,1733534061701; forceNewPlan=false, retain=false 2024-12-07T01:14:24,622 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=75c927364905f2c6bffff16328d8aaf5, ASSIGN 2024-12-07T01:14:24,624 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=75c927364905f2c6bffff16328d8aaf5, ASSIGN; state=OFFLINE, location=ec1863dc21e5,42915,1733534061580; forceNewPlan=false, retain=false 2024-12-07T01:14:24,626 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=aa3dca7b6c799eb77565fb0e20ef1c15, ASSIGN 2024-12-07T01:14:24,627 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=aa3dca7b6c799eb77565fb0e20ef1c15, ASSIGN; state=OFFLINE, location=ec1863dc21e5,40693,1733534061744; forceNewPlan=false, retain=false 2024-12-07T01:14:24,629 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=5d3306d206c48a1c7060e49d4fe00ad5, ASSIGN 2024-12-07T01:14:24,630 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=5d3306d206c48a1c7060e49d4fe00ad5, ASSIGN; state=OFFLINE, location=ec1863dc21e5,36079,1733534061701; forceNewPlan=false, retain=false 2024-12-07T01:14:24,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46141 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-07T01:14:24,631 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=9c426833151f173ef0d81d9c0ae11767, ASSIGN 2024-12-07T01:14:24,633 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=9c426833151f173ef0d81d9c0ae11767, ASSIGN; state=OFFLINE, location=ec1863dc21e5,42915,1733534061580; forceNewPlan=false, retain=false 2024-12-07T01:14:24,634 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c0f43810347933724c9e6319b11ae5cc, ASSIGN 2024-12-07T01:14:24,635 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=c0f43810347933724c9e6319b11ae5cc, ASSIGN; state=OFFLINE, location=ec1863dc21e5,40693,1733534061744; forceNewPlan=false, retain=false 2024-12-07T01:14:24,636 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=37d131a6ac713be02dc73a29079210f5, ASSIGN 2024-12-07T01:14:24,638 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=37d131a6ac713be02dc73a29079210f5, ASSIGN; state=OFFLINE, location=ec1863dc21e5,36079,1733534061701; forceNewPlan=false, retain=false 2024-12-07T01:14:24,639 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c0a570532487ea0ebbc51cbec9ff7e37, ASSIGN 2024-12-07T01:14:24,640 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=c0a570532487ea0ebbc51cbec9ff7e37, ASSIGN; state=OFFLINE, location=ec1863dc21e5,42915,1733534061580; forceNewPlan=false, retain=false 2024-12-07T01:14:24,641 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=af843c9df2505d2eeee3c18062e9a2a6, ASSIGN 2024-12-07T01:14:24,643 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=af843c9df2505d2eeee3c18062e9a2a6, ASSIGN; state=OFFLINE, location=ec1863dc21e5,40693,1733534061744; forceNewPlan=false, retain=false 2024-12-07T01:14:24,644 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=6d8689bfa03ba7aed23b15e5ff5d23ac, ASSIGN 2024-12-07T01:14:24,645 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=6d8689bfa03ba7aed23b15e5ff5d23ac, ASSIGN; state=OFFLINE, location=ec1863dc21e5,36079,1733534061701; forceNewPlan=false, retain=false 2024-12-07T01:14:24,647 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=0980fd6969c12f09b843d772c626dfa5, ASSIGN 2024-12-07T01:14:24,648 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=0980fd6969c12f09b843d772c626dfa5, ASSIGN; state=OFFLINE, location=ec1863dc21e5,42915,1733534061580; forceNewPlan=false, retain=false 2024-12-07T01:14:24,650 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=976bbe8d7afc3b1e02af11d1b515e82c, ASSIGN 2024-12-07T01:14:24,652 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=976bbe8d7afc3b1e02af11d1b515e82c, ASSIGN; state=OFFLINE, location=ec1863dc21e5,40693,1733534061744; forceNewPlan=false, retain=false 2024-12-07T01:14:24,735 INFO [ec1863dc21e5:46141 {}] balancer.BaseLoadBalancer(388): Reassigned 26 regions. 26 retained the pre-restart assignment. 2024-12-07T01:14:24,737 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=73ec7ce1c93542e31d99b7b4474a9750, regionState=OPENING, regionLocation=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:24,739 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=8cd2040dc039667d68bc1a24f5f2322f, regionState=OPENING, regionLocation=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:24,740 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=3f148a9f8c5bea63f4744afb36553ef7, regionState=OPENING, regionLocation=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:24,741 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=856730db89c49e65ccb40b5967ccf99d, regionState=OPENING, regionLocation=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:24,742 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=3fe1b6a546d269c366ae3f13263e1a2a, regionState=OPENING, regionLocation=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:24,742 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=73ec7ce1c93542e31d99b7b4474a9750, ASSIGN because future has completed 2024-12-07T01:14:24,743 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=75c927364905f2c6bffff16328d8aaf5, regionState=OPENING, regionLocation=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:24,744 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure 73ec7ce1c93542e31d99b7b4474a9750, server=ec1863dc21e5,42915,1733534061580}] 2024-12-07T01:14:24,745 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=8cd2040dc039667d68bc1a24f5f2322f, ASSIGN because future has completed 2024-12-07T01:14:24,747 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=3f148a9f8c5bea63f4744afb36553ef7, ASSIGN because future has completed 2024-12-07T01:14:24,749 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=856730db89c49e65ccb40b5967ccf99d, ASSIGN because future has completed 2024-12-07T01:14:24,751 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=3fe1b6a546d269c366ae3f13263e1a2a, ASSIGN because future has completed 2024-12-07T01:14:24,753 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3fe1b6a546d269c366ae3f13263e1a2a, server=ec1863dc21e5,42915,1733534061580}] 2024-12-07T01:14:24,753 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=75c927364905f2c6bffff16328d8aaf5, ASSIGN because future has completed 2024-12-07T01:14:24,759 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure 75c927364905f2c6bffff16328d8aaf5, server=ec1863dc21e5,42915,1733534061580}] 2024-12-07T01:14:24,764 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure 856730db89c49e65ccb40b5967ccf99d, server=ec1863dc21e5,42915,1733534061580}] 2024-12-07T01:14:24,769 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3f148a9f8c5bea63f4744afb36553ef7, server=ec1863dc21e5,42915,1733534061580}] 2024-12-07T01:14:24,773 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8cd2040dc039667d68bc1a24f5f2322f, server=ec1863dc21e5,42915,1733534061580}] 2024-12-07T01:14:24,779 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=9c426833151f173ef0d81d9c0ae11767, regionState=OPENING, regionLocation=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:24,780 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=c0a570532487ea0ebbc51cbec9ff7e37, regionState=OPENING, regionLocation=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:24,781 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=0980fd6969c12f09b843d772c626dfa5, regionState=OPENING, regionLocation=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:24,782 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=b59f40eb016096571b13ae445f262449, regionState=OPENING, regionLocation=ec1863dc21e5,40693,1733534061744 2024-12-07T01:14:24,783 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=76cd5c1764feda4c992319b218572cef, regionState=OPENING, regionLocation=ec1863dc21e5,40693,1733534061744 2024-12-07T01:14:24,784 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=1da36e78b320d37341113b4185c91b26, regionState=OPENING, regionLocation=ec1863dc21e5,40693,1733534061744 2024-12-07T01:14:24,784 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=9c426833151f173ef0d81d9c0ae11767, ASSIGN because future has completed 2024-12-07T01:14:24,785 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=c66f402dd9e22edaa5996a3b7446808c, regionState=OPENING, regionLocation=ec1863dc21e5,40693,1733534061744 2024-12-07T01:14:24,786 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9c426833151f173ef0d81d9c0ae11767, server=ec1863dc21e5,42915,1733534061580}] 2024-12-07T01:14:24,789 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=c0a570532487ea0ebbc51cbec9ff7e37, ASSIGN because future has completed 2024-12-07T01:14:24,789 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure c0a570532487ea0ebbc51cbec9ff7e37, server=ec1863dc21e5,42915,1733534061580}] 2024-12-07T01:14:24,793 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=0980fd6969c12f09b843d772c626dfa5, ASSIGN because future has completed 2024-12-07T01:14:24,795 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=b59f40eb016096571b13ae445f262449, ASSIGN because future has completed 2024-12-07T01:14:24,795 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure b59f40eb016096571b13ae445f262449, server=ec1863dc21e5,40693,1733534061744}] 2024-12-07T01:14:24,796 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=76cd5c1764feda4c992319b218572cef, ASSIGN because future has completed 2024-12-07T01:14:24,798 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=1da36e78b320d37341113b4185c91b26, ASSIGN because future has completed 2024-12-07T01:14:24,799 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=c66f402dd9e22edaa5996a3b7446808c, ASSIGN because future has completed 2024-12-07T01:14:24,800 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure c66f402dd9e22edaa5996a3b7446808c, server=ec1863dc21e5,40693,1733534061744}] 2024-12-07T01:14:24,805 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1da36e78b320d37341113b4185c91b26, server=ec1863dc21e5,40693,1733534061744}] 2024-12-07T01:14:24,810 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=42, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 76cd5c1764feda4c992319b218572cef, server=ec1863dc21e5,40693,1733534061744}] 2024-12-07T01:14:24,814 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0980fd6969c12f09b843d772c626dfa5, server=ec1863dc21e5,42915,1733534061580}] 2024-12-07T01:14:24,824 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=327d2bdcc20803fcfd1e61553f30068b, regionState=OPENING, regionLocation=ec1863dc21e5,40693,1733534061744 2024-12-07T01:14:24,825 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=aa3dca7b6c799eb77565fb0e20ef1c15, regionState=OPENING, regionLocation=ec1863dc21e5,40693,1733534061744 2024-12-07T01:14:24,825 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=c0f43810347933724c9e6319b11ae5cc, regionState=OPENING, regionLocation=ec1863dc21e5,40693,1733534061744 2024-12-07T01:14:24,826 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=af843c9df2505d2eeee3c18062e9a2a6, regionState=OPENING, regionLocation=ec1863dc21e5,40693,1733534061744 2024-12-07T01:14:24,827 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=976bbe8d7afc3b1e02af11d1b515e82c, regionState=OPENING, regionLocation=ec1863dc21e5,40693,1733534061744 2024-12-07T01:14:24,827 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=1cf1ce3ee0822ff957b8ffa13f56df7d, regionState=OPENING, regionLocation=ec1863dc21e5,36079,1733534061701 2024-12-07T01:14:24,827 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=327d2bdcc20803fcfd1e61553f30068b, ASSIGN because future has completed 2024-12-07T01:14:24,828 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=44, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure 327d2bdcc20803fcfd1e61553f30068b, server=ec1863dc21e5,40693,1733534061744}] 2024-12-07T01:14:24,830 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=aa3dca7b6c799eb77565fb0e20ef1c15, ASSIGN because future has completed 2024-12-07T01:14:24,831 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=c0f43810347933724c9e6319b11ae5cc, ASSIGN because future has completed 2024-12-07T01:14:24,833 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=af843c9df2505d2eeee3c18062e9a2a6, ASSIGN because future has completed 2024-12-07T01:14:24,834 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure af843c9df2505d2eeee3c18062e9a2a6, server=ec1863dc21e5,40693,1733534061744}] 2024-12-07T01:14:24,834 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=976bbe8d7afc3b1e02af11d1b515e82c, ASSIGN because future has completed 2024-12-07T01:14:24,836 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=1cf1ce3ee0822ff957b8ffa13f56df7d, ASSIGN because future has completed 2024-12-07T01:14:24,838 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=46, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1cf1ce3ee0822ff957b8ffa13f56df7d, server=ec1863dc21e5,36079,1733534061701}] 2024-12-07T01:14:24,843 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 976bbe8d7afc3b1e02af11d1b515e82c, server=ec1863dc21e5,40693,1733534061744}] 2024-12-07T01:14:24,847 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=48, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure c0f43810347933724c9e6319b11ae5cc, server=ec1863dc21e5,40693,1733534061744}] 2024-12-07T01:14:24,852 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure aa3dca7b6c799eb77565fb0e20ef1c15, server=ec1863dc21e5,40693,1733534061744}] 2024-12-07T01:14:24,856 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=80bacdcc81a44a51d622e9d873892ea3, regionState=OPENING, regionLocation=ec1863dc21e5,36079,1733534061701 2024-12-07T01:14:24,857 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=fa4e20ed49e59ee4cf63b6d84515872f, regionState=OPENING, regionLocation=ec1863dc21e5,36079,1733534061701 2024-12-07T01:14:24,857 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=324bdaae20264b3880845c3082304e57, regionState=OPENING, regionLocation=ec1863dc21e5,36079,1733534061701 2024-12-07T01:14:24,857 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=d2b1eb41dda894f4f7b826a2826a9c7b, regionState=OPENING, regionLocation=ec1863dc21e5,36079,1733534061701 2024-12-07T01:14:24,858 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=5d3306d206c48a1c7060e49d4fe00ad5, regionState=OPENING, regionLocation=ec1863dc21e5,36079,1733534061701 2024-12-07T01:14:24,859 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=37d131a6ac713be02dc73a29079210f5, regionState=OPENING, regionLocation=ec1863dc21e5,36079,1733534061701 2024-12-07T01:14:24,859 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=6d8689bfa03ba7aed23b15e5ff5d23ac, regionState=OPENING, regionLocation=ec1863dc21e5,36079,1733534061701 2024-12-07T01:14:24,859 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=80bacdcc81a44a51d622e9d873892ea3, ASSIGN because future has completed 2024-12-07T01:14:24,860 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=50, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure 80bacdcc81a44a51d622e9d873892ea3, server=ec1863dc21e5,36079,1733534061701}] 2024-12-07T01:14:24,861 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=fa4e20ed49e59ee4cf63b6d84515872f, ASSIGN because future has completed 2024-12-07T01:14:24,862 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=324bdaae20264b3880845c3082304e57, ASSIGN because future has completed 2024-12-07T01:14:24,864 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=d2b1eb41dda894f4f7b826a2826a9c7b, ASSIGN because future has completed 2024-12-07T01:14:24,865 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure d2b1eb41dda894f4f7b826a2826a9c7b, server=ec1863dc21e5,36079,1733534061701}] 2024-12-07T01:14:24,865 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=5d3306d206c48a1c7060e49d4fe00ad5, ASSIGN because future has completed 2024-12-07T01:14:24,867 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=37d131a6ac713be02dc73a29079210f5, ASSIGN because future has completed 2024-12-07T01:14:24,868 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=6d8689bfa03ba7aed23b15e5ff5d23ac, ASSIGN because future has completed 2024-12-07T01:14:24,869 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=52, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6d8689bfa03ba7aed23b15e5ff5d23ac, server=ec1863dc21e5,36079,1733534061701}] 2024-12-07T01:14:24,873 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 37d131a6ac713be02dc73a29079210f5, server=ec1863dc21e5,36079,1733534061701}] 2024-12-07T01:14:24,877 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=54, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5d3306d206c48a1c7060e49d4fe00ad5, server=ec1863dc21e5,36079,1733534061701}] 2024-12-07T01:14:24,881 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure 324bdaae20264b3880845c3082304e57, server=ec1863dc21e5,36079,1733534061701}] 2024-12-07T01:14:24,884 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure fa4e20ed49e59ee4cf63b6d84515872f, server=ec1863dc21e5,36079,1733534061701}] 2024-12-07T01:14:24,899 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-07T01:14:24,901 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52131, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-07T01:14:24,906 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(132): Open test,hhh,1733534064285.9c426833151f173ef0d81d9c0ae11767. 2024-12-07T01:14:24,907 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7752): Opening region: {ENCODED => 9c426833151f173ef0d81d9c0ae11767, NAME => 'test,hhh,1733534064285.9c426833151f173ef0d81d9c0ae11767.', STARTKEY => 'hhh', ENDKEY => 'iii'} 2024-12-07T01:14:24,907 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 9c426833151f173ef0d81d9c0ae11767 2024-12-07T01:14:24,907 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(898): Instantiated test,hhh,1733534064285.9c426833151f173ef0d81d9c0ae11767.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T01:14:24,907 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7794): checking encryption for 9c426833151f173ef0d81d9c0ae11767 2024-12-07T01:14:24,908 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7797): checking classloading for 9c426833151f173ef0d81d9c0ae11767 2024-12-07T01:14:24,910 INFO [StoreOpener-9c426833151f173ef0d81d9c0ae11767-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 9c426833151f173ef0d81d9c0ae11767 2024-12-07T01:14:24,911 INFO [StoreOpener-9c426833151f173ef0d81d9c0ae11767-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9c426833151f173ef0d81d9c0ae11767 columnFamilyName family 2024-12-07T01:14:24,912 DEBUG [StoreOpener-9c426833151f173ef0d81d9c0ae11767-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:14:24,912 INFO [StoreOpener-9c426833151f173ef0d81d9c0ae11767-1 {}] regionserver.HStore(327): Store=9c426833151f173ef0d81d9c0ae11767/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T01:14:24,913 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1038): replaying wal for 9c426833151f173ef0d81d9c0ae11767 2024-12-07T01:14:24,914 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/default/test/9c426833151f173ef0d81d9c0ae11767 2024-12-07T01:14:24,914 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/default/test/9c426833151f173ef0d81d9c0ae11767 2024-12-07T01:14:24,915 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1048): stopping wal replay for 9c426833151f173ef0d81d9c0ae11767 2024-12-07T01:14:24,915 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1060): Cleaning up temporary data for 9c426833151f173ef0d81d9c0ae11767 2024-12-07T01:14:24,917 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1093): writing seq id for 9c426833151f173ef0d81d9c0ae11767 2024-12-07T01:14:24,920 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/default/test/9c426833151f173ef0d81d9c0ae11767/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-07T01:14:24,921 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1114): Opened 9c426833151f173ef0d81d9c0ae11767; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66758610, jitterRate=-0.005219191312789917}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-07T01:14:24,921 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9c426833151f173ef0d81d9c0ae11767 2024-12-07T01:14:24,921 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1006): Region open journal for 9c426833151f173ef0d81d9c0ae11767: Running coprocessor pre-open hook at 1733534064908Writing region info on filesystem at 1733534064908Initializing all the Stores at 1733534064909 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534064909Cleaning up temporary data from old regions at 1733534064915 (+6 ms)Running coprocessor post-open hooks at 1733534064921 (+6 ms)Region opened successfully at 1733534064921 2024-12-07T01:14:24,923 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2236): Post open deploy tasks for test,hhh,1733534064285.9c426833151f173ef0d81d9c0ae11767., pid=37, masterSystemTime=1733534064899 2024-12-07T01:14:24,926 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2266): Finished post open deploy task for test,hhh,1733534064285.9c426833151f173ef0d81d9c0ae11767. 2024-12-07T01:14:24,926 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(153): Opened test,hhh,1733534064285.9c426833151f173ef0d81d9c0ae11767. 2024-12-07T01:14:24,926 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(132): Open test,bbb,1733534064285.0980fd6969c12f09b843d772c626dfa5. 2024-12-07T01:14:24,927 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7752): Opening region: {ENCODED => 0980fd6969c12f09b843d772c626dfa5, NAME => 'test,bbb,1733534064285.0980fd6969c12f09b843d772c626dfa5.', STARTKEY => 'bbb', ENDKEY => 'ccc'} 2024-12-07T01:14:24,927 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 0980fd6969c12f09b843d772c626dfa5 2024-12-07T01:14:24,927 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=9c426833151f173ef0d81d9c0ae11767, regionState=OPEN, openSeqNum=2, regionLocation=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:24,927 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(898): Instantiated test,bbb,1733534064285.0980fd6969c12f09b843d772c626dfa5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T01:14:24,927 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7794): checking encryption for 0980fd6969c12f09b843d772c626dfa5 2024-12-07T01:14:24,927 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7797): checking classloading for 0980fd6969c12f09b843d772c626dfa5 2024-12-07T01:14:24,930 INFO [StoreOpener-0980fd6969c12f09b843d772c626dfa5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 0980fd6969c12f09b843d772c626dfa5 2024-12-07T01:14:24,931 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9c426833151f173ef0d81d9c0ae11767, server=ec1863dc21e5,42915,1733534061580 because future has completed 2024-12-07T01:14:24,933 INFO [StoreOpener-0980fd6969c12f09b843d772c626dfa5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0980fd6969c12f09b843d772c626dfa5 columnFamilyName family 2024-12-07T01:14:24,933 DEBUG [StoreOpener-0980fd6969c12f09b843d772c626dfa5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:14:24,934 INFO [StoreOpener-0980fd6969c12f09b843d772c626dfa5-1 {}] regionserver.HStore(327): Store=0980fd6969c12f09b843d772c626dfa5/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T01:14:24,934 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1038): replaying wal for 0980fd6969c12f09b843d772c626dfa5 2024-12-07T01:14:24,935 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/default/test/0980fd6969c12f09b843d772c626dfa5 2024-12-07T01:14:24,936 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=12 2024-12-07T01:14:24,936 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=12, state=SUCCESS, hasLock=false; OpenRegionProcedure 9c426833151f173ef0d81d9c0ae11767, server=ec1863dc21e5,42915,1733534061580 in 147 msec 2024-12-07T01:14:24,936 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/default/test/0980fd6969c12f09b843d772c626dfa5 2024-12-07T01:14:24,936 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1048): stopping wal replay for 0980fd6969c12f09b843d772c626dfa5 2024-12-07T01:14:24,936 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1060): Cleaning up temporary data for 0980fd6969c12f09b843d772c626dfa5 2024-12-07T01:14:24,938 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=9c426833151f173ef0d81d9c0ae11767, ASSIGN in 367 msec 2024-12-07T01:14:24,939 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1093): writing seq id for 0980fd6969c12f09b843d772c626dfa5 2024-12-07T01:14:24,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46141 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-07T01:14:24,943 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/default/test/0980fd6969c12f09b843d772c626dfa5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-07T01:14:24,943 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1114): Opened 0980fd6969c12f09b843d772c626dfa5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69398748, jitterRate=0.034121930599212646}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-07T01:14:24,943 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0980fd6969c12f09b843d772c626dfa5 2024-12-07T01:14:24,944 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1006): Region open journal for 0980fd6969c12f09b843d772c626dfa5: Running coprocessor pre-open hook at 1733534064928Writing region info on filesystem at 1733534064928Initializing all the Stores at 1733534064930 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534064930Cleaning up temporary data from old regions at 1733534064936 (+6 ms)Running coprocessor post-open hooks at 1733534064944 (+8 ms)Region opened successfully at 1733534064944 2024-12-07T01:14:24,945 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2236): Post open deploy tasks for test,bbb,1733534064285.0980fd6969c12f09b843d772c626dfa5., pid=43, masterSystemTime=1733534064899 2024-12-07T01:14:24,947 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2266): Finished post open deploy task for test,bbb,1733534064285.0980fd6969c12f09b843d772c626dfa5. 2024-12-07T01:14:24,947 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(153): Opened test,bbb,1733534064285.0980fd6969c12f09b843d772c626dfa5. 2024-12-07T01:14:24,948 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(132): Open test,nnn,1733534064285.3fe1b6a546d269c366ae3f13263e1a2a. 2024-12-07T01:14:24,948 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7752): Opening region: {ENCODED => 3fe1b6a546d269c366ae3f13263e1a2a, NAME => 'test,nnn,1733534064285.3fe1b6a546d269c366ae3f13263e1a2a.', STARTKEY => 'nnn', ENDKEY => 'ooo'} 2024-12-07T01:14:24,948 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=0980fd6969c12f09b843d772c626dfa5, regionState=OPEN, openSeqNum=2, regionLocation=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:24,948 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 3fe1b6a546d269c366ae3f13263e1a2a 2024-12-07T01:14:24,948 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(898): Instantiated test,nnn,1733534064285.3fe1b6a546d269c366ae3f13263e1a2a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T01:14:24,948 DEBUG [RSProcedureDispatcher-pool-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-07T01:14:24,948 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7794): checking encryption for 3fe1b6a546d269c366ae3f13263e1a2a 2024-12-07T01:14:24,948 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7797): checking classloading for 3fe1b6a546d269c366ae3f13263e1a2a 2024-12-07T01:14:24,950 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35999, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-07T01:14:24,951 INFO [StoreOpener-3fe1b6a546d269c366ae3f13263e1a2a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 3fe1b6a546d269c366ae3f13263e1a2a 2024-12-07T01:14:24,952 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=43, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0980fd6969c12f09b843d772c626dfa5, server=ec1863dc21e5,42915,1733534061580 because future has completed 2024-12-07T01:14:24,953 INFO [StoreOpener-3fe1b6a546d269c366ae3f13263e1a2a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3fe1b6a546d269c366ae3f13263e1a2a columnFamilyName family 2024-12-07T01:14:24,953 DEBUG [StoreOpener-3fe1b6a546d269c366ae3f13263e1a2a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:14:24,954 INFO [StoreOpener-3fe1b6a546d269c366ae3f13263e1a2a-1 {}] regionserver.HStore(327): Store=3fe1b6a546d269c366ae3f13263e1a2a/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T01:14:24,954 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1038): replaying wal for 3fe1b6a546d269c366ae3f13263e1a2a 2024-12-07T01:14:24,955 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(132): Open test,vvv,1733534064285.76cd5c1764feda4c992319b218572cef. 2024-12-07T01:14:24,955 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7752): Opening region: {ENCODED => 76cd5c1764feda4c992319b218572cef, NAME => 'test,vvv,1733534064285.76cd5c1764feda4c992319b218572cef.', STARTKEY => 'vvv', ENDKEY => 'www'} 2024-12-07T01:14:24,956 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/default/test/3fe1b6a546d269c366ae3f13263e1a2a 2024-12-07T01:14:24,956 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 76cd5c1764feda4c992319b218572cef 2024-12-07T01:14:24,956 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(898): Instantiated test,vvv,1733534064285.76cd5c1764feda4c992319b218572cef.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T01:14:24,956 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7794): checking encryption for 76cd5c1764feda4c992319b218572cef 2024-12-07T01:14:24,956 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7797): checking classloading for 76cd5c1764feda4c992319b218572cef 2024-12-07T01:14:24,957 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/default/test/3fe1b6a546d269c366ae3f13263e1a2a 2024-12-07T01:14:24,957 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1048): stopping wal replay for 3fe1b6a546d269c366ae3f13263e1a2a 2024-12-07T01:14:24,957 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1060): Cleaning up temporary data for 3fe1b6a546d269c366ae3f13263e1a2a 2024-12-07T01:14:24,957 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=6 2024-12-07T01:14:24,958 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=6, state=SUCCESS, hasLock=false; OpenRegionProcedure 0980fd6969c12f09b843d772c626dfa5, server=ec1863dc21e5,42915,1733534061580 in 140 msec 2024-12-07T01:14:24,958 INFO [StoreOpener-76cd5c1764feda4c992319b218572cef-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 76cd5c1764feda4c992319b218572cef 2024-12-07T01:14:24,960 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1093): writing seq id for 3fe1b6a546d269c366ae3f13263e1a2a 2024-12-07T01:14:24,961 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=0980fd6969c12f09b843d772c626dfa5, ASSIGN in 389 msec 2024-12-07T01:14:24,961 INFO [StoreOpener-76cd5c1764feda4c992319b218572cef-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 76cd5c1764feda4c992319b218572cef columnFamilyName family 2024-12-07T01:14:24,962 DEBUG [StoreOpener-76cd5c1764feda4c992319b218572cef-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:14:24,962 INFO [StoreOpener-76cd5c1764feda4c992319b218572cef-1 {}] regionserver.HStore(327): Store=76cd5c1764feda4c992319b218572cef/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T01:14:24,962 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1038): replaying wal for 76cd5c1764feda4c992319b218572cef 2024-12-07T01:14:24,963 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/default/test/3fe1b6a546d269c366ae3f13263e1a2a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-07T01:14:24,964 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/default/test/76cd5c1764feda4c992319b218572cef 2024-12-07T01:14:24,963 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1114): Opened 3fe1b6a546d269c366ae3f13263e1a2a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64651996, jitterRate=-0.036610186100006104}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-07T01:14:24,964 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3fe1b6a546d269c366ae3f13263e1a2a 2024-12-07T01:14:24,964 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1006): Region open journal for 3fe1b6a546d269c366ae3f13263e1a2a: Running coprocessor pre-open hook at 1733534064949Writing region info on filesystem at 1733534064949Initializing all the Stores at 1733534064950 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534064950Cleaning up temporary data from old regions at 1733534064957 (+7 ms)Running coprocessor post-open hooks at 1733534064964 (+7 ms)Region opened successfully at 1733534064964 2024-12-07T01:14:24,964 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/default/test/76cd5c1764feda4c992319b218572cef 2024-12-07T01:14:24,965 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1048): stopping wal replay for 76cd5c1764feda4c992319b218572cef 2024-12-07T01:14:24,965 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1060): Cleaning up temporary data for 76cd5c1764feda4c992319b218572cef 2024-12-07T01:14:24,965 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2236): Post open deploy tasks for test,nnn,1733534064285.3fe1b6a546d269c366ae3f13263e1a2a., pid=32, masterSystemTime=1733534064899 2024-12-07T01:14:24,967 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1093): writing seq id for 76cd5c1764feda4c992319b218572cef 2024-12-07T01:14:24,968 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2266): Finished post open deploy task for test,nnn,1733534064285.3fe1b6a546d269c366ae3f13263e1a2a. 2024-12-07T01:14:24,968 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(153): Opened test,nnn,1733534064285.3fe1b6a546d269c366ae3f13263e1a2a. 2024-12-07T01:14:24,968 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(132): Open test,qqq,1733534064285.856730db89c49e65ccb40b5967ccf99d. 2024-12-07T01:14:24,968 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7752): Opening region: {ENCODED => 856730db89c49e65ccb40b5967ccf99d, NAME => 'test,qqq,1733534064285.856730db89c49e65ccb40b5967ccf99d.', STARTKEY => 'qqq', ENDKEY => 'rrr'} 2024-12-07T01:14:24,969 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 856730db89c49e65ccb40b5967ccf99d 2024-12-07T01:14:24,969 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=3fe1b6a546d269c366ae3f13263e1a2a, regionState=OPEN, openSeqNum=2, regionLocation=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:24,969 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(898): Instantiated test,qqq,1733534064285.856730db89c49e65ccb40b5967ccf99d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T01:14:24,969 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7794): checking encryption for 856730db89c49e65ccb40b5967ccf99d 2024-12-07T01:14:24,969 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7797): checking classloading for 856730db89c49e65ccb40b5967ccf99d 2024-12-07T01:14:24,971 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/default/test/76cd5c1764feda4c992319b218572cef/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-07T01:14:24,971 INFO [StoreOpener-856730db89c49e65ccb40b5967ccf99d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 856730db89c49e65ccb40b5967ccf99d 2024-12-07T01:14:24,972 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3fe1b6a546d269c366ae3f13263e1a2a, server=ec1863dc21e5,42915,1733534061580 because future has completed 2024-12-07T01:14:24,972 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1114): Opened 76cd5c1764feda4c992319b218572cef; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65633500, jitterRate=-0.021984636783599854}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-07T01:14:24,972 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 76cd5c1764feda4c992319b218572cef 2024-12-07T01:14:24,972 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1006): Region open journal for 76cd5c1764feda4c992319b218572cef: Running coprocessor pre-open hook at 1733534064956Writing region info on filesystem at 1733534064956Initializing all the Stores at 1733534064958 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534064958Cleaning up temporary data from old regions at 1733534064965 (+7 ms)Running coprocessor post-open hooks at 1733534064972 (+7 ms)Region opened successfully at 1733534064972 2024-12-07T01:14:24,973 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2236): Post open deploy tasks for test,vvv,1733534064285.76cd5c1764feda4c992319b218572cef., pid=42, masterSystemTime=1733534064948 2024-12-07T01:14:24,973 INFO [StoreOpener-856730db89c49e65ccb40b5967ccf99d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 856730db89c49e65ccb40b5967ccf99d columnFamilyName family 2024-12-07T01:14:24,974 DEBUG [StoreOpener-856730db89c49e65ccb40b5967ccf99d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:14:24,974 INFO [StoreOpener-856730db89c49e65ccb40b5967ccf99d-1 {}] regionserver.HStore(327): Store=856730db89c49e65ccb40b5967ccf99d/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T01:14:24,975 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1038): replaying wal for 856730db89c49e65ccb40b5967ccf99d 2024-12-07T01:14:24,976 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2266): Finished post open deploy task for test,vvv,1733534064285.76cd5c1764feda4c992319b218572cef. 2024-12-07T01:14:24,976 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(153): Opened test,vvv,1733534064285.76cd5c1764feda4c992319b218572cef. 2024-12-07T01:14:24,977 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(132): Open test,mmm,1733534064285.327d2bdcc20803fcfd1e61553f30068b. 2024-12-07T01:14:24,977 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7752): Opening region: {ENCODED => 327d2bdcc20803fcfd1e61553f30068b, NAME => 'test,mmm,1733534064285.327d2bdcc20803fcfd1e61553f30068b.', STARTKEY => 'mmm', ENDKEY => 'nnn'} 2024-12-07T01:14:24,977 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/default/test/856730db89c49e65ccb40b5967ccf99d 2024-12-07T01:14:24,977 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 327d2bdcc20803fcfd1e61553f30068b 2024-12-07T01:14:24,977 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(898): Instantiated test,mmm,1733534064285.327d2bdcc20803fcfd1e61553f30068b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T01:14:24,977 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7794): checking encryption for 327d2bdcc20803fcfd1e61553f30068b 2024-12-07T01:14:24,977 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7797): checking classloading for 327d2bdcc20803fcfd1e61553f30068b 2024-12-07T01:14:24,978 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/default/test/856730db89c49e65ccb40b5967ccf99d 2024-12-07T01:14:24,978 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=18 2024-12-07T01:14:24,978 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=18, state=SUCCESS, hasLock=false; OpenRegionProcedure 3fe1b6a546d269c366ae3f13263e1a2a, server=ec1863dc21e5,42915,1733534061580 in 221 msec 2024-12-07T01:14:24,978 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1048): stopping wal replay for 856730db89c49e65ccb40b5967ccf99d 2024-12-07T01:14:24,978 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1060): Cleaning up temporary data for 856730db89c49e65ccb40b5967ccf99d 2024-12-07T01:14:24,979 INFO [StoreOpener-327d2bdcc20803fcfd1e61553f30068b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 327d2bdcc20803fcfd1e61553f30068b 2024-12-07T01:14:24,981 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=3fe1b6a546d269c366ae3f13263e1a2a, ASSIGN in 409 msec 2024-12-07T01:14:24,981 INFO [StoreOpener-327d2bdcc20803fcfd1e61553f30068b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 327d2bdcc20803fcfd1e61553f30068b columnFamilyName family 2024-12-07T01:14:24,981 DEBUG [StoreOpener-327d2bdcc20803fcfd1e61553f30068b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:14:24,982 INFO [StoreOpener-327d2bdcc20803fcfd1e61553f30068b-1 {}] regionserver.HStore(327): Store=327d2bdcc20803fcfd1e61553f30068b/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T01:14:24,982 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1093): writing seq id for 856730db89c49e65ccb40b5967ccf99d 2024-12-07T01:14:24,982 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1038): replaying wal for 327d2bdcc20803fcfd1e61553f30068b 2024-12-07T01:14:24,982 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=76cd5c1764feda4c992319b218572cef, regionState=OPEN, openSeqNum=2, regionLocation=ec1863dc21e5,40693,1733534061744 2024-12-07T01:14:24,983 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/default/test/327d2bdcc20803fcfd1e61553f30068b 2024-12-07T01:14:24,983 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/default/test/327d2bdcc20803fcfd1e61553f30068b 2024-12-07T01:14:24,984 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1048): stopping wal replay for 327d2bdcc20803fcfd1e61553f30068b 2024-12-07T01:14:24,984 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1060): Cleaning up temporary data for 327d2bdcc20803fcfd1e61553f30068b 2024-12-07T01:14:24,985 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/default/test/856730db89c49e65ccb40b5967ccf99d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-07T01:14:24,986 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=42, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 76cd5c1764feda4c992319b218572cef, server=ec1863dc21e5,40693,1733534061744 because future has completed 2024-12-07T01:14:24,986 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1114): Opened 856730db89c49e65ccb40b5967ccf99d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72784704, jitterRate=0.08457660675048828}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-07T01:14:24,986 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 856730db89c49e65ccb40b5967ccf99d 2024-12-07T01:14:24,986 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1006): Region open journal for 856730db89c49e65ccb40b5967ccf99d: Running coprocessor pre-open hook at 1733534064969Writing region info on filesystem at 1733534064969Initializing all the Stores at 1733534064971 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534064971Cleaning up temporary data from old regions at 1733534064979 (+8 ms)Running coprocessor post-open hooks at 1733534064986 (+7 ms)Region opened successfully at 1733534064986 2024-12-07T01:14:24,987 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2236): Post open deploy tasks for test,qqq,1733534064285.856730db89c49e65ccb40b5967ccf99d., pid=34, masterSystemTime=1733534064899 2024-12-07T01:14:24,987 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1093): writing seq id for 327d2bdcc20803fcfd1e61553f30068b 2024-12-07T01:14:24,990 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2266): Finished post open deploy task for test,qqq,1733534064285.856730db89c49e65ccb40b5967ccf99d. 2024-12-07T01:14:24,990 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(153): Opened test,qqq,1733534064285.856730db89c49e65ccb40b5967ccf99d. 2024-12-07T01:14:24,990 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open test,kkk,1733534064285.75c927364905f2c6bffff16328d8aaf5. 2024-12-07T01:14:24,990 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/default/test/327d2bdcc20803fcfd1e61553f30068b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-07T01:14:24,990 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=42, resume processing ppid=26 2024-12-07T01:14:24,990 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => 75c927364905f2c6bffff16328d8aaf5, NAME => 'test,kkk,1733534064285.75c927364905f2c6bffff16328d8aaf5.', STARTKEY => 'kkk', ENDKEY => 'lll'} 2024-12-07T01:14:24,991 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, ppid=26, state=SUCCESS, hasLock=false; OpenRegionProcedure 76cd5c1764feda4c992319b218572cef, server=ec1863dc21e5,40693,1733534061744 in 178 msec 2024-12-07T01:14:24,991 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 75c927364905f2c6bffff16328d8aaf5 2024-12-07T01:14:24,991 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated test,kkk,1733534064285.75c927364905f2c6bffff16328d8aaf5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T01:14:24,991 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for 75c927364905f2c6bffff16328d8aaf5 2024-12-07T01:14:24,991 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for 75c927364905f2c6bffff16328d8aaf5 2024-12-07T01:14:24,991 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1114): Opened 327d2bdcc20803fcfd1e61553f30068b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59022338, jitterRate=-0.12049862742424011}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-07T01:14:24,991 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 327d2bdcc20803fcfd1e61553f30068b 2024-12-07T01:14:24,992 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1006): Region open journal for 327d2bdcc20803fcfd1e61553f30068b: Running coprocessor pre-open hook at 1733534064978Writing region info on filesystem at 1733534064978Initializing all the Stores at 1733534064979 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534064979Cleaning up temporary data from old regions at 1733534064984 (+5 ms)Running coprocessor post-open hooks at 1733534064991 (+7 ms)Region opened successfully at 1733534064991 2024-12-07T01:14:24,993 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2236): Post open deploy tasks for test,mmm,1733534064285.327d2bdcc20803fcfd1e61553f30068b., pid=44, masterSystemTime=1733534064948 2024-12-07T01:14:24,994 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=76cd5c1764feda4c992319b218572cef, ASSIGN in 422 msec 2024-12-07T01:14:24,995 INFO [StoreOpener-75c927364905f2c6bffff16328d8aaf5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 75c927364905f2c6bffff16328d8aaf5 2024-12-07T01:14:24,995 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(132): Open test,uuu,1733534064285.80bacdcc81a44a51d622e9d873892ea3. 2024-12-07T01:14:24,995 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=856730db89c49e65ccb40b5967ccf99d, regionState=OPEN, openSeqNum=2, regionLocation=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:24,995 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7752): Opening region: {ENCODED => 80bacdcc81a44a51d622e9d873892ea3, NAME => 'test,uuu,1733534064285.80bacdcc81a44a51d622e9d873892ea3.', STARTKEY => 'uuu', ENDKEY => 'vvv'} 2024-12-07T01:14:24,996 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2266): Finished post open deploy task for test,mmm,1733534064285.327d2bdcc20803fcfd1e61553f30068b. 2024-12-07T01:14:24,996 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(153): Opened test,mmm,1733534064285.327d2bdcc20803fcfd1e61553f30068b. 2024-12-07T01:14:24,996 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 80bacdcc81a44a51d622e9d873892ea3 2024-12-07T01:14:24,996 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(898): Instantiated test,uuu,1733534064285.80bacdcc81a44a51d622e9d873892ea3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T01:14:24,996 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(132): Open test,sss,1733534064285.1da36e78b320d37341113b4185c91b26. 2024-12-07T01:14:24,996 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7794): checking encryption for 80bacdcc81a44a51d622e9d873892ea3 2024-12-07T01:14:24,996 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7752): Opening region: {ENCODED => 1da36e78b320d37341113b4185c91b26, NAME => 'test,sss,1733534064285.1da36e78b320d37341113b4185c91b26.', STARTKEY => 'sss', ENDKEY => 'ttt'} 2024-12-07T01:14:24,997 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7797): checking classloading for 80bacdcc81a44a51d622e9d873892ea3 2024-12-07T01:14:24,997 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 1da36e78b320d37341113b4185c91b26 2024-12-07T01:14:24,997 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(898): Instantiated test,sss,1733534064285.1da36e78b320d37341113b4185c91b26.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T01:14:24,998 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7794): checking encryption for 1da36e78b320d37341113b4185c91b26 2024-12-07T01:14:24,998 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7797): checking classloading for 1da36e78b320d37341113b4185c91b26 2024-12-07T01:14:24,998 INFO [StoreOpener-75c927364905f2c6bffff16328d8aaf5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 75c927364905f2c6bffff16328d8aaf5 columnFamilyName family 2024-12-07T01:14:24,998 DEBUG [StoreOpener-75c927364905f2c6bffff16328d8aaf5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:14:24,999 INFO [StoreOpener-75c927364905f2c6bffff16328d8aaf5-1 {}] regionserver.HStore(327): Store=75c927364905f2c6bffff16328d8aaf5/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T01:14:24,999 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure 856730db89c49e65ccb40b5967ccf99d, server=ec1863dc21e5,42915,1733534061580 because future has completed 2024-12-07T01:14:24,999 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for 75c927364905f2c6bffff16328d8aaf5 2024-12-07T01:14:25,000 INFO [StoreOpener-80bacdcc81a44a51d622e9d873892ea3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 80bacdcc81a44a51d622e9d873892ea3 2024-12-07T01:14:25,000 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=327d2bdcc20803fcfd1e61553f30068b, regionState=OPEN, openSeqNum=2, regionLocation=ec1863dc21e5,40693,1733534061744 2024-12-07T01:14:25,000 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/default/test/75c927364905f2c6bffff16328d8aaf5 2024-12-07T01:14:25,000 INFO [StoreOpener-1da36e78b320d37341113b4185c91b26-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 1da36e78b320d37341113b4185c91b26 2024-12-07T01:14:25,001 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/default/test/75c927364905f2c6bffff16328d8aaf5 2024-12-07T01:14:25,002 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for 75c927364905f2c6bffff16328d8aaf5 2024-12-07T01:14:25,002 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for 75c927364905f2c6bffff16328d8aaf5 2024-12-07T01:14:25,002 INFO [StoreOpener-80bacdcc81a44a51d622e9d873892ea3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 80bacdcc81a44a51d622e9d873892ea3 columnFamilyName family 2024-12-07T01:14:25,002 DEBUG [StoreOpener-80bacdcc81a44a51d622e9d873892ea3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:14:25,003 INFO [StoreOpener-1da36e78b320d37341113b4185c91b26-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1da36e78b320d37341113b4185c91b26 columnFamilyName family 2024-12-07T01:14:25,003 DEBUG [StoreOpener-1da36e78b320d37341113b4185c91b26-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:14:25,003 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=44, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure 327d2bdcc20803fcfd1e61553f30068b, server=ec1863dc21e5,40693,1733534061744 because future has completed 2024-12-07T01:14:25,003 INFO [StoreOpener-80bacdcc81a44a51d622e9d873892ea3-1 {}] regionserver.HStore(327): Store=80bacdcc81a44a51d622e9d873892ea3/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T01:14:25,003 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1038): replaying wal for 80bacdcc81a44a51d622e9d873892ea3 2024-12-07T01:14:25,004 INFO [StoreOpener-1da36e78b320d37341113b4185c91b26-1 {}] regionserver.HStore(327): Store=1da36e78b320d37341113b4185c91b26/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T01:14:25,004 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1038): replaying wal for 1da36e78b320d37341113b4185c91b26 2024-12-07T01:14:25,004 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for 75c927364905f2c6bffff16328d8aaf5 2024-12-07T01:14:25,005 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/default/test/80bacdcc81a44a51d622e9d873892ea3 2024-12-07T01:14:25,005 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/default/test/1da36e78b320d37341113b4185c91b26 2024-12-07T01:14:25,005 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/default/test/80bacdcc81a44a51d622e9d873892ea3 2024-12-07T01:14:25,005 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/default/test/1da36e78b320d37341113b4185c91b26 2024-12-07T01:14:25,006 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1048): stopping wal replay for 1da36e78b320d37341113b4185c91b26 2024-12-07T01:14:25,006 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1060): Cleaning up temporary data for 1da36e78b320d37341113b4185c91b26 2024-12-07T01:14:25,006 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1048): stopping wal replay for 80bacdcc81a44a51d622e9d873892ea3 2024-12-07T01:14:25,006 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1060): Cleaning up temporary data for 80bacdcc81a44a51d622e9d873892ea3 2024-12-07T01:14:25,007 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=21 2024-12-07T01:14:25,007 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=21, state=SUCCESS, hasLock=false; OpenRegionProcedure 856730db89c49e65ccb40b5967ccf99d, server=ec1863dc21e5,42915,1733534061580 in 239 msec 2024-12-07T01:14:25,007 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/default/test/75c927364905f2c6bffff16328d8aaf5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-07T01:14:25,008 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened 75c927364905f2c6bffff16328d8aaf5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61136959, jitterRate=-0.08898831903934479}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-07T01:14:25,008 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 75c927364905f2c6bffff16328d8aaf5 2024-12-07T01:14:25,009 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for 75c927364905f2c6bffff16328d8aaf5: Running coprocessor pre-open hook at 1733534064991Writing region info on filesystem at 1733534064991Initializing all the Stores at 1733534064994 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534064995 (+1 ms)Cleaning up temporary data from old regions at 1733534065002 (+7 ms)Running coprocessor post-open hooks at 1733534065009 (+7 ms)Region opened successfully at 1733534065009 2024-12-07T01:14:25,009 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1093): writing seq id for 80bacdcc81a44a51d622e9d873892ea3 2024-12-07T01:14:25,009 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1093): writing seq id for 1da36e78b320d37341113b4185c91b26 2024-12-07T01:14:25,009 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=856730db89c49e65ccb40b5967ccf99d, ASSIGN in 438 msec 2024-12-07T01:14:25,010 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for test,kkk,1733534064285.75c927364905f2c6bffff16328d8aaf5., pid=33, masterSystemTime=1733534064899 2024-12-07T01:14:25,012 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/default/test/80bacdcc81a44a51d622e9d873892ea3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-07T01:14:25,013 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for test,kkk,1733534064285.75c927364905f2c6bffff16328d8aaf5. 2024-12-07T01:14:25,012 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/default/test/1da36e78b320d37341113b4185c91b26/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-07T01:14:25,013 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened test,kkk,1733534064285.75c927364905f2c6bffff16328d8aaf5. 2024-12-07T01:14:25,013 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(132): Open test,ttt,1733534064285.3f148a9f8c5bea63f4744afb36553ef7. 2024-12-07T01:14:25,013 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7752): Opening region: {ENCODED => 3f148a9f8c5bea63f4744afb36553ef7, NAME => 'test,ttt,1733534064285.3f148a9f8c5bea63f4744afb36553ef7.', STARTKEY => 'ttt', ENDKEY => 'uuu'} 2024-12-07T01:14:25,013 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1114): Opened 1da36e78b320d37341113b4185c91b26; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69802993, jitterRate=0.04014565050601959}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-07T01:14:25,013 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1114): Opened 80bacdcc81a44a51d622e9d873892ea3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64401886, jitterRate=-0.04033711552619934}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-07T01:14:25,014 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1da36e78b320d37341113b4185c91b26 2024-12-07T01:14:25,014 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 3f148a9f8c5bea63f4744afb36553ef7 2024-12-07T01:14:25,014 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 80bacdcc81a44a51d622e9d873892ea3 2024-12-07T01:14:25,014 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1006): Region open journal for 1da36e78b320d37341113b4185c91b26: Running coprocessor pre-open hook at 1733534064998Writing region info on filesystem at 1733534064998Initializing all the Stores at 1733534065000 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534065000Cleaning up temporary data from old regions at 1733534065006 (+6 ms)Running coprocessor post-open hooks at 1733534065014 (+8 ms)Region opened successfully at 1733534065014 2024-12-07T01:14:25,014 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1006): Region open journal for 80bacdcc81a44a51d622e9d873892ea3: Running coprocessor pre-open hook at 1733534064997Writing region info on filesystem at 1733534064997Initializing all the Stores at 1733534064999 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534064999Cleaning up temporary data from old regions at 1733534065006 (+7 ms)Running coprocessor post-open hooks at 1733534065014 (+8 ms)Region opened successfully at 1733534065014 2024-12-07T01:14:25,014 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(898): Instantiated test,ttt,1733534064285.3f148a9f8c5bea63f4744afb36553ef7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T01:14:25,014 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7794): checking encryption for 3f148a9f8c5bea63f4744afb36553ef7 2024-12-07T01:14:25,014 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7797): checking classloading for 3f148a9f8c5bea63f4744afb36553ef7 2024-12-07T01:14:25,014 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=44, resume processing ppid=17 2024-12-07T01:14:25,014 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, ppid=17, state=SUCCESS, hasLock=false; OpenRegionProcedure 327d2bdcc20803fcfd1e61553f30068b, server=ec1863dc21e5,40693,1733534061744 in 183 msec 2024-12-07T01:14:25,015 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2236): Post open deploy tasks for test,uuu,1733534064285.80bacdcc81a44a51d622e9d873892ea3., pid=50, masterSystemTime=1733534064990 2024-12-07T01:14:25,015 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2236): Post open deploy tasks for test,sss,1733534064285.1da36e78b320d37341113b4185c91b26., pid=41, masterSystemTime=1733534064948 2024-12-07T01:14:25,016 INFO [StoreOpener-3f148a9f8c5bea63f4744afb36553ef7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 3f148a9f8c5bea63f4744afb36553ef7 2024-12-07T01:14:25,017 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=327d2bdcc20803fcfd1e61553f30068b, ASSIGN in 446 msec 2024-12-07T01:14:25,017 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2266): Finished post open deploy task for test,sss,1733534064285.1da36e78b320d37341113b4185c91b26. 2024-12-07T01:14:25,017 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(153): Opened test,sss,1733534064285.1da36e78b320d37341113b4185c91b26. 2024-12-07T01:14:25,018 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(132): Open test,ggg,1733534064285.c0f43810347933724c9e6319b11ae5cc. 2024-12-07T01:14:25,018 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7752): Opening region: {ENCODED => c0f43810347933724c9e6319b11ae5cc, NAME => 'test,ggg,1733534064285.c0f43810347933724c9e6319b11ae5cc.', STARTKEY => 'ggg', ENDKEY => 'hhh'} 2024-12-07T01:14:25,018 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test c0f43810347933724c9e6319b11ae5cc 2024-12-07T01:14:25,018 INFO [StoreOpener-3f148a9f8c5bea63f4744afb36553ef7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3f148a9f8c5bea63f4744afb36553ef7 columnFamilyName family 2024-12-07T01:14:25,018 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(898): Instantiated test,ggg,1733534064285.c0f43810347933724c9e6319b11ae5cc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T01:14:25,018 DEBUG [StoreOpener-3f148a9f8c5bea63f4744afb36553ef7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:14:25,018 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7794): checking encryption for c0f43810347933724c9e6319b11ae5cc 2024-12-07T01:14:25,018 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7797): checking classloading for c0f43810347933724c9e6319b11ae5cc 2024-12-07T01:14:25,019 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=1da36e78b320d37341113b4185c91b26, regionState=OPEN, openSeqNum=2, regionLocation=ec1863dc21e5,40693,1733534061744 2024-12-07T01:14:25,019 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2266): Finished post open deploy task for test,uuu,1733534064285.80bacdcc81a44a51d622e9d873892ea3. 2024-12-07T01:14:25,019 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(153): Opened test,uuu,1733534064285.80bacdcc81a44a51d622e9d873892ea3. 2024-12-07T01:14:25,019 INFO [StoreOpener-3f148a9f8c5bea63f4744afb36553ef7-1 {}] regionserver.HStore(327): Store=3f148a9f8c5bea63f4744afb36553ef7/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T01:14:25,019 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(132): Open test,iii,1733534064285.5d3306d206c48a1c7060e49d4fe00ad5. 2024-12-07T01:14:25,019 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1038): replaying wal for 3f148a9f8c5bea63f4744afb36553ef7 2024-12-07T01:14:25,019 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7752): Opening region: {ENCODED => 5d3306d206c48a1c7060e49d4fe00ad5, NAME => 'test,iii,1733534064285.5d3306d206c48a1c7060e49d4fe00ad5.', STARTKEY => 'iii', ENDKEY => 'jjj'} 2024-12-07T01:14:25,020 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 5d3306d206c48a1c7060e49d4fe00ad5 2024-12-07T01:14:25,020 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(898): Instantiated test,iii,1733534064285.5d3306d206c48a1c7060e49d4fe00ad5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T01:14:25,020 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7794): checking encryption for 5d3306d206c48a1c7060e49d4fe00ad5 2024-12-07T01:14:25,020 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7797): checking classloading for 5d3306d206c48a1c7060e49d4fe00ad5 2024-12-07T01:14:25,020 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/default/test/3f148a9f8c5bea63f4744afb36553ef7 2024-12-07T01:14:25,021 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/default/test/3f148a9f8c5bea63f4744afb36553ef7 2024-12-07T01:14:25,022 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1048): stopping wal replay for 3f148a9f8c5bea63f4744afb36553ef7 2024-12-07T01:14:25,022 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1060): Cleaning up temporary data for 3f148a9f8c5bea63f4744afb36553ef7 2024-12-07T01:14:25,022 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1da36e78b320d37341113b4185c91b26, server=ec1863dc21e5,40693,1733534061744 because future has completed 2024-12-07T01:14:25,022 INFO [StoreOpener-5d3306d206c48a1c7060e49d4fe00ad5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 5d3306d206c48a1c7060e49d4fe00ad5 2024-12-07T01:14:25,023 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=80bacdcc81a44a51d622e9d873892ea3, regionState=OPEN, openSeqNum=2, regionLocation=ec1863dc21e5,36079,1733534061701 2024-12-07T01:14:25,023 INFO [StoreOpener-c0f43810347933724c9e6319b11ae5cc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region c0f43810347933724c9e6319b11ae5cc 2024-12-07T01:14:25,024 INFO [StoreOpener-5d3306d206c48a1c7060e49d4fe00ad5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5d3306d206c48a1c7060e49d4fe00ad5 columnFamilyName family 2024-12-07T01:14:25,024 DEBUG [StoreOpener-5d3306d206c48a1c7060e49d4fe00ad5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:14:25,024 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1093): writing seq id for 3f148a9f8c5bea63f4744afb36553ef7 2024-12-07T01:14:25,025 INFO [StoreOpener-c0f43810347933724c9e6319b11ae5cc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c0f43810347933724c9e6319b11ae5cc columnFamilyName family 2024-12-07T01:14:25,025 DEBUG [StoreOpener-c0f43810347933724c9e6319b11ae5cc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:14:25,025 INFO [StoreOpener-5d3306d206c48a1c7060e49d4fe00ad5-1 {}] regionserver.HStore(327): Store=5d3306d206c48a1c7060e49d4fe00ad5/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T01:14:25,026 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1038): replaying wal for 5d3306d206c48a1c7060e49d4fe00ad5 2024-12-07T01:14:25,026 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=50, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure 80bacdcc81a44a51d622e9d873892ea3, server=ec1863dc21e5,36079,1733534061701 because future has completed 2024-12-07T01:14:25,026 INFO [StoreOpener-c0f43810347933724c9e6319b11ae5cc-1 {}] regionserver.HStore(327): Store=c0f43810347933724c9e6319b11ae5cc/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T01:14:25,027 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/default/test/5d3306d206c48a1c7060e49d4fe00ad5 2024-12-07T01:14:25,027 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1038): replaying wal for c0f43810347933724c9e6319b11ae5cc 2024-12-07T01:14:25,027 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/default/test/5d3306d206c48a1c7060e49d4fe00ad5 2024-12-07T01:14:25,028 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/default/test/c0f43810347933724c9e6319b11ae5cc 2024-12-07T01:14:25,028 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1048): stopping wal replay for 5d3306d206c48a1c7060e49d4fe00ad5 2024-12-07T01:14:25,028 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1060): Cleaning up temporary data for 5d3306d206c48a1c7060e49d4fe00ad5 2024-12-07T01:14:25,028 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/default/test/3f148a9f8c5bea63f4744afb36553ef7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-07T01:14:25,028 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/default/test/c0f43810347933724c9e6319b11ae5cc 2024-12-07T01:14:25,029 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1048): stopping wal replay for c0f43810347933724c9e6319b11ae5cc 2024-12-07T01:14:25,029 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1060): Cleaning up temporary data for c0f43810347933724c9e6319b11ae5cc 2024-12-07T01:14:25,029 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1114): Opened 3f148a9f8c5bea63f4744afb36553ef7; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62183713, jitterRate=-0.07339046895503998}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-07T01:14:25,029 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3f148a9f8c5bea63f4744afb36553ef7 2024-12-07T01:14:25,029 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1006): Region open journal for 3f148a9f8c5bea63f4744afb36553ef7: Running coprocessor pre-open hook at 1733534065014Writing region info on filesystem at 1733534065014Initializing all the Stores at 1733534065016 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534065016Cleaning up temporary data from old regions at 1733534065022 (+6 ms)Running coprocessor post-open hooks at 1733534065029 (+7 ms)Region opened successfully at 1733534065029 2024-12-07T01:14:25,031 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1093): writing seq id for 5d3306d206c48a1c7060e49d4fe00ad5 2024-12-07T01:14:25,032 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=23 2024-12-07T01:14:25,032 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1093): writing seq id for c0f43810347933724c9e6319b11ae5cc 2024-12-07T01:14:25,032 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=23, state=SUCCESS, hasLock=false; OpenRegionProcedure 1da36e78b320d37341113b4185c91b26, server=ec1863dc21e5,40693,1733534061744 in 222 msec 2024-12-07T01:14:25,035 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/default/test/c0f43810347933724c9e6319b11ae5cc/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-07T01:14:25,035 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/default/test/5d3306d206c48a1c7060e49d4fe00ad5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-07T01:14:25,035 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=1da36e78b320d37341113b4185c91b26, ASSIGN in 463 msec 2024-12-07T01:14:25,035 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ttt,1733534064285.3f148a9f8c5bea63f4744afb36553ef7., pid=35, masterSystemTime=1733534064899 2024-12-07T01:14:25,036 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1114): Opened c0f43810347933724c9e6319b11ae5cc; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71653543, jitterRate=0.06772099435329437}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-07T01:14:25,036 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1114): Opened 5d3306d206c48a1c7060e49d4fe00ad5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69351892, jitterRate=0.03342372179031372}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-07T01:14:25,036 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c0f43810347933724c9e6319b11ae5cc 2024-12-07T01:14:25,036 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5d3306d206c48a1c7060e49d4fe00ad5 2024-12-07T01:14:25,036 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1006): Region open journal for c0f43810347933724c9e6319b11ae5cc: Running coprocessor pre-open hook at 1733534065019Writing region info on filesystem at 1733534065019Initializing all the Stores at 1733534065023 (+4 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534065023Cleaning up temporary data from old regions at 1733534065029 (+6 ms)Running coprocessor post-open hooks at 1733534065036 (+7 ms)Region opened successfully at 1733534065036 2024-12-07T01:14:25,036 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1006): Region open journal for 5d3306d206c48a1c7060e49d4fe00ad5: Running coprocessor pre-open hook at 1733534065020Writing region info on filesystem at 1733534065020Initializing all the Stores at 1733534065022 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534065022Cleaning up temporary data from old regions at 1733534065028 (+6 ms)Running coprocessor post-open hooks at 1733534065036 (+8 ms)Region opened successfully at 1733534065036 2024-12-07T01:14:25,037 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2236): Post open deploy tasks for test,iii,1733534064285.5d3306d206c48a1c7060e49d4fe00ad5., pid=54, masterSystemTime=1733534064990 2024-12-07T01:14:25,037 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ggg,1733534064285.c0f43810347933724c9e6319b11ae5cc., pid=48, masterSystemTime=1733534064948 2024-12-07T01:14:25,038 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ttt,1733534064285.3f148a9f8c5bea63f4744afb36553ef7. 2024-12-07T01:14:25,038 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(153): Opened test,ttt,1733534064285.3f148a9f8c5bea63f4744afb36553ef7. 2024-12-07T01:14:25,038 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(132): Open test,zzz,1733534064285.73ec7ce1c93542e31d99b7b4474a9750. 2024-12-07T01:14:25,038 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7752): Opening region: {ENCODED => 73ec7ce1c93542e31d99b7b4474a9750, NAME => 'test,zzz,1733534064285.73ec7ce1c93542e31d99b7b4474a9750.', STARTKEY => 'zzz', ENDKEY => ''} 2024-12-07T01:14:25,039 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 73ec7ce1c93542e31d99b7b4474a9750 2024-12-07T01:14:25,039 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(898): Instantiated test,zzz,1733534064285.73ec7ce1c93542e31d99b7b4474a9750.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T01:14:25,039 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7794): checking encryption for 73ec7ce1c93542e31d99b7b4474a9750 2024-12-07T01:14:25,039 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7797): checking classloading for 73ec7ce1c93542e31d99b7b4474a9750 2024-12-07T01:14:25,039 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ggg,1733534064285.c0f43810347933724c9e6319b11ae5cc. 2024-12-07T01:14:25,040 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(153): Opened test,ggg,1733534064285.c0f43810347933724c9e6319b11ae5cc. 2024-12-07T01:14:25,040 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(132): Open test,ppp,1733534064285.c66f402dd9e22edaa5996a3b7446808c. 2024-12-07T01:14:25,040 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7752): Opening region: {ENCODED => c66f402dd9e22edaa5996a3b7446808c, NAME => 'test,ppp,1733534064285.c66f402dd9e22edaa5996a3b7446808c.', STARTKEY => 'ppp', ENDKEY => 'qqq'} 2024-12-07T01:14:25,040 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test c66f402dd9e22edaa5996a3b7446808c 2024-12-07T01:14:25,040 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(898): Instantiated test,ppp,1733534064285.c66f402dd9e22edaa5996a3b7446808c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T01:14:25,041 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7794): checking encryption for c66f402dd9e22edaa5996a3b7446808c 2024-12-07T01:14:25,041 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7797): checking classloading for c66f402dd9e22edaa5996a3b7446808c 2024-12-07T01:14:25,041 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=50, resume processing ppid=25 2024-12-07T01:14:25,041 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, ppid=25, state=SUCCESS, hasLock=false; OpenRegionProcedure 80bacdcc81a44a51d622e9d873892ea3, server=ec1863dc21e5,36079,1733534061701 in 176 msec 2024-12-07T01:14:25,041 INFO [StoreOpener-73ec7ce1c93542e31d99b7b4474a9750-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 73ec7ce1c93542e31d99b7b4474a9750 2024-12-07T01:14:25,041 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2266): Finished post open deploy task for test,iii,1733534064285.5d3306d206c48a1c7060e49d4fe00ad5. 2024-12-07T01:14:25,041 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(153): Opened test,iii,1733534064285.5d3306d206c48a1c7060e49d4fe00ad5. 2024-12-07T01:14:25,041 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(132): Open test,ccc,1733534064285.6d8689bfa03ba7aed23b15e5ff5d23ac. 2024-12-07T01:14:25,042 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7752): Opening region: {ENCODED => 6d8689bfa03ba7aed23b15e5ff5d23ac, NAME => 'test,ccc,1733534064285.6d8689bfa03ba7aed23b15e5ff5d23ac.', STARTKEY => 'ccc', ENDKEY => 'ddd'} 2024-12-07T01:14:25,042 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 6d8689bfa03ba7aed23b15e5ff5d23ac 2024-12-07T01:14:25,042 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(898): Instantiated test,ccc,1733534064285.6d8689bfa03ba7aed23b15e5ff5d23ac.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T01:14:25,042 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7794): checking encryption for 6d8689bfa03ba7aed23b15e5ff5d23ac 2024-12-07T01:14:25,042 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7797): checking classloading for 6d8689bfa03ba7aed23b15e5ff5d23ac 2024-12-07T01:14:25,042 INFO [StoreOpener-c66f402dd9e22edaa5996a3b7446808c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region c66f402dd9e22edaa5996a3b7446808c 2024-12-07T01:14:25,043 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=5d3306d206c48a1c7060e49d4fe00ad5, regionState=OPEN, openSeqNum=2, regionLocation=ec1863dc21e5,36079,1733534061701 2024-12-07T01:14:25,043 INFO [StoreOpener-73ec7ce1c93542e31d99b7b4474a9750-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 73ec7ce1c93542e31d99b7b4474a9750 columnFamilyName family 2024-12-07T01:14:25,043 DEBUG [StoreOpener-73ec7ce1c93542e31d99b7b4474a9750-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:14:25,044 INFO [StoreOpener-73ec7ce1c93542e31d99b7b4474a9750-1 {}] regionserver.HStore(327): Store=73ec7ce1c93542e31d99b7b4474a9750/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T01:14:25,044 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1038): replaying wal for 73ec7ce1c93542e31d99b7b4474a9750 2024-12-07T01:14:25,044 INFO [StoreOpener-c66f402dd9e22edaa5996a3b7446808c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c66f402dd9e22edaa5996a3b7446808c columnFamilyName family 2024-12-07T01:14:25,045 DEBUG [StoreOpener-c66f402dd9e22edaa5996a3b7446808c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:14:25,045 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/default/test/73ec7ce1c93542e31d99b7b4474a9750 2024-12-07T01:14:25,045 INFO [StoreOpener-c66f402dd9e22edaa5996a3b7446808c-1 {}] regionserver.HStore(327): Store=c66f402dd9e22edaa5996a3b7446808c/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T01:14:25,046 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1038): replaying wal for c66f402dd9e22edaa5996a3b7446808c 2024-12-07T01:14:25,047 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/default/test/c66f402dd9e22edaa5996a3b7446808c 2024-12-07T01:14:25,047 INFO [StoreOpener-6d8689bfa03ba7aed23b15e5ff5d23ac-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 6d8689bfa03ba7aed23b15e5ff5d23ac 2024-12-07T01:14:25,047 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/default/test/73ec7ce1c93542e31d99b7b4474a9750 2024-12-07T01:14:25,047 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/default/test/c66f402dd9e22edaa5996a3b7446808c 2024-12-07T01:14:25,048 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1048): stopping wal replay for c66f402dd9e22edaa5996a3b7446808c 2024-12-07T01:14:25,048 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1060): Cleaning up temporary data for c66f402dd9e22edaa5996a3b7446808c 2024-12-07T01:14:25,048 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=54, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5d3306d206c48a1c7060e49d4fe00ad5, server=ec1863dc21e5,36079,1733534061701 because future has completed 2024-12-07T01:14:25,048 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1048): stopping wal replay for 73ec7ce1c93542e31d99b7b4474a9750 2024-12-07T01:14:25,048 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1060): Cleaning up temporary data for 73ec7ce1c93542e31d99b7b4474a9750 2024-12-07T01:14:25,050 INFO [StoreOpener-6d8689bfa03ba7aed23b15e5ff5d23ac-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6d8689bfa03ba7aed23b15e5ff5d23ac columnFamilyName family 2024-12-07T01:14:25,050 DEBUG [StoreOpener-6d8689bfa03ba7aed23b15e5ff5d23ac-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:14:25,050 INFO [StoreOpener-6d8689bfa03ba7aed23b15e5ff5d23ac-1 {}] regionserver.HStore(327): Store=6d8689bfa03ba7aed23b15e5ff5d23ac/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T01:14:25,051 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1038): replaying wal for 6d8689bfa03ba7aed23b15e5ff5d23ac 2024-12-07T01:14:25,052 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1093): writing seq id for c66f402dd9e22edaa5996a3b7446808c 2024-12-07T01:14:25,052 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1093): writing seq id for 73ec7ce1c93542e31d99b7b4474a9750 2024-12-07T01:14:25,052 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/default/test/6d8689bfa03ba7aed23b15e5ff5d23ac 2024-12-07T01:14:25,052 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=80bacdcc81a44a51d622e9d873892ea3, ASSIGN in 476 msec 2024-12-07T01:14:25,052 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/default/test/6d8689bfa03ba7aed23b15e5ff5d23ac 2024-12-07T01:14:25,053 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1048): stopping wal replay for 6d8689bfa03ba7aed23b15e5ff5d23ac 2024-12-07T01:14:25,053 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1060): Cleaning up temporary data for 6d8689bfa03ba7aed23b15e5ff5d23ac 2024-12-07T01:14:25,055 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/default/test/c66f402dd9e22edaa5996a3b7446808c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-07T01:14:25,056 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1114): Opened c66f402dd9e22edaa5996a3b7446808c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60608564, jitterRate=-0.09686201810836792}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-07T01:14:25,056 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c66f402dd9e22edaa5996a3b7446808c 2024-12-07T01:14:25,056 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1006): Region open journal for c66f402dd9e22edaa5996a3b7446808c: Running coprocessor pre-open hook at 1733534065041Writing region info on filesystem at 1733534065041Initializing all the Stores at 1733534065042 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534065042Cleaning up temporary data from old regions at 1733534065048 (+6 ms)Running coprocessor post-open hooks at 1733534065056 (+8 ms)Region opened successfully at 1733534065056 2024-12-07T01:14:25,058 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ppp,1733534064285.c66f402dd9e22edaa5996a3b7446808c., pid=40, masterSystemTime=1733534064948 2024-12-07T01:14:25,058 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1093): writing seq id for 6d8689bfa03ba7aed23b15e5ff5d23ac 2024-12-07T01:14:25,059 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=54, resume processing ppid=13 2024-12-07T01:14:25,059 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, ppid=13, state=SUCCESS, hasLock=false; OpenRegionProcedure 5d3306d206c48a1c7060e49d4fe00ad5, server=ec1863dc21e5,36079,1733534061701 in 177 msec 2024-12-07T01:14:25,059 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/default/test/73ec7ce1c93542e31d99b7b4474a9750/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-07T01:14:25,060 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1114): Opened 73ec7ce1c93542e31d99b7b4474a9750; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64882644, jitterRate=-0.03317326307296753}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-07T01:14:25,060 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 73ec7ce1c93542e31d99b7b4474a9750 2024-12-07T01:14:25,061 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1006): Region open journal for 73ec7ce1c93542e31d99b7b4474a9750: Running coprocessor pre-open hook at 1733534065039Writing region info on filesystem at 1733534065039Initializing all the Stores at 1733534065041 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534065041Cleaning up temporary data from old regions at 1733534065048 (+7 ms)Running coprocessor post-open hooks at 1733534065060 (+12 ms)Region opened successfully at 1733534065061 (+1 ms) 2024-12-07T01:14:25,061 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ppp,1733534064285.c66f402dd9e22edaa5996a3b7446808c. 2024-12-07T01:14:25,061 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(153): Opened test,ppp,1733534064285.c66f402dd9e22edaa5996a3b7446808c. 2024-12-07T01:14:25,061 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(132): Open test,ddd,1733534064285.af843c9df2505d2eeee3c18062e9a2a6. 2024-12-07T01:14:25,061 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7752): Opening region: {ENCODED => af843c9df2505d2eeee3c18062e9a2a6, NAME => 'test,ddd,1733534064285.af843c9df2505d2eeee3c18062e9a2a6.', STARTKEY => 'ddd', ENDKEY => 'eee'} 2024-12-07T01:14:25,062 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test af843c9df2505d2eeee3c18062e9a2a6 2024-12-07T01:14:25,062 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(898): Instantiated test,ddd,1733534064285.af843c9df2505d2eeee3c18062e9a2a6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T01:14:25,062 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7794): checking encryption for af843c9df2505d2eeee3c18062e9a2a6 2024-12-07T01:14:25,062 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2236): Post open deploy tasks for test,zzz,1733534064285.73ec7ce1c93542e31d99b7b4474a9750., pid=31, masterSystemTime=1733534064899 2024-12-07T01:14:25,062 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/default/test/6d8689bfa03ba7aed23b15e5ff5d23ac/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-07T01:14:25,062 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7797): checking classloading for af843c9df2505d2eeee3c18062e9a2a6 2024-12-07T01:14:25,062 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=5d3306d206c48a1c7060e49d4fe00ad5, ASSIGN in 490 msec 2024-12-07T01:14:25,063 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1114): Opened 6d8689bfa03ba7aed23b15e5ff5d23ac; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72594919, jitterRate=0.0817485898733139}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-07T01:14:25,063 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6d8689bfa03ba7aed23b15e5ff5d23ac 2024-12-07T01:14:25,063 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1006): Region open journal for 6d8689bfa03ba7aed23b15e5ff5d23ac: Running coprocessor pre-open hook at 1733534065042Writing region info on filesystem at 1733534065042Initializing all the Stores at 1733534065044 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534065044Cleaning up temporary data from old regions at 1733534065053 (+9 ms)Running coprocessor post-open hooks at 1733534065063 (+10 ms)Region opened successfully at 1733534065063 2024-12-07T01:14:25,064 INFO [StoreOpener-af843c9df2505d2eeee3c18062e9a2a6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region af843c9df2505d2eeee3c18062e9a2a6 2024-12-07T01:14:25,064 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ccc,1733534064285.6d8689bfa03ba7aed23b15e5ff5d23ac., pid=52, masterSystemTime=1733534064990 2024-12-07T01:14:25,065 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=c66f402dd9e22edaa5996a3b7446808c, regionState=OPEN, openSeqNum=2, regionLocation=ec1863dc21e5,40693,1733534061744 2024-12-07T01:14:25,065 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2266): Finished post open deploy task for test,zzz,1733534064285.73ec7ce1c93542e31d99b7b4474a9750. 2024-12-07T01:14:25,065 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(153): Opened test,zzz,1733534064285.73ec7ce1c93542e31d99b7b4474a9750. 2024-12-07T01:14:25,065 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(132): Open test,www,1733534064285.8cd2040dc039667d68bc1a24f5f2322f. 2024-12-07T01:14:25,066 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7752): Opening region: {ENCODED => 8cd2040dc039667d68bc1a24f5f2322f, NAME => 'test,www,1733534064285.8cd2040dc039667d68bc1a24f5f2322f.', STARTKEY => 'www', ENDKEY => 'xxx'} 2024-12-07T01:14:25,066 INFO [StoreOpener-af843c9df2505d2eeee3c18062e9a2a6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region af843c9df2505d2eeee3c18062e9a2a6 columnFamilyName family 2024-12-07T01:14:25,066 DEBUG [StoreOpener-af843c9df2505d2eeee3c18062e9a2a6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:14:25,066 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 8cd2040dc039667d68bc1a24f5f2322f 2024-12-07T01:14:25,066 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(898): Instantiated test,www,1733534064285.8cd2040dc039667d68bc1a24f5f2322f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T01:14:25,066 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7794): checking encryption for 8cd2040dc039667d68bc1a24f5f2322f 2024-12-07T01:14:25,066 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7797): checking classloading for 8cd2040dc039667d68bc1a24f5f2322f 2024-12-07T01:14:25,067 INFO [StoreOpener-af843c9df2505d2eeee3c18062e9a2a6-1 {}] regionserver.HStore(327): Store=af843c9df2505d2eeee3c18062e9a2a6/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T01:14:25,067 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1038): replaying wal for af843c9df2505d2eeee3c18062e9a2a6 2024-12-07T01:14:25,068 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ccc,1733534064285.6d8689bfa03ba7aed23b15e5ff5d23ac. 2024-12-07T01:14:25,068 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(153): Opened test,ccc,1733534064285.6d8689bfa03ba7aed23b15e5ff5d23ac. 2024-12-07T01:14:25,068 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/default/test/af843c9df2505d2eeee3c18062e9a2a6 2024-12-07T01:14:25,068 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=40, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure c66f402dd9e22edaa5996a3b7446808c, server=ec1863dc21e5,40693,1733534061744 because future has completed 2024-12-07T01:14:25,068 INFO [StoreOpener-8cd2040dc039667d68bc1a24f5f2322f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 8cd2040dc039667d68bc1a24f5f2322f 2024-12-07T01:14:25,068 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(132): Open test,xxx,1733534064285.1cf1ce3ee0822ff957b8ffa13f56df7d. 2024-12-07T01:14:25,069 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7752): Opening region: {ENCODED => 1cf1ce3ee0822ff957b8ffa13f56df7d, NAME => 'test,xxx,1733534064285.1cf1ce3ee0822ff957b8ffa13f56df7d.', STARTKEY => 'xxx', ENDKEY => 'yyy'} 2024-12-07T01:14:25,069 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/default/test/af843c9df2505d2eeee3c18062e9a2a6 2024-12-07T01:14:25,069 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 1cf1ce3ee0822ff957b8ffa13f56df7d 2024-12-07T01:14:25,069 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=73ec7ce1c93542e31d99b7b4474a9750, regionState=OPEN, openSeqNum=2, regionLocation=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:25,069 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(898): Instantiated test,xxx,1733534064285.1cf1ce3ee0822ff957b8ffa13f56df7d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T01:14:25,069 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7794): checking encryption for 1cf1ce3ee0822ff957b8ffa13f56df7d 2024-12-07T01:14:25,069 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7797): checking classloading for 1cf1ce3ee0822ff957b8ffa13f56df7d 2024-12-07T01:14:25,070 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1048): stopping wal replay for af843c9df2505d2eeee3c18062e9a2a6 2024-12-07T01:14:25,070 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1060): Cleaning up temporary data for af843c9df2505d2eeee3c18062e9a2a6 2024-12-07T01:14:25,070 INFO [StoreOpener-8cd2040dc039667d68bc1a24f5f2322f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8cd2040dc039667d68bc1a24f5f2322f columnFamilyName family 2024-12-07T01:14:25,070 DEBUG [StoreOpener-8cd2040dc039667d68bc1a24f5f2322f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:14:25,071 INFO [StoreOpener-1cf1ce3ee0822ff957b8ffa13f56df7d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 1cf1ce3ee0822ff957b8ffa13f56df7d 2024-12-07T01:14:25,071 INFO [StoreOpener-8cd2040dc039667d68bc1a24f5f2322f-1 {}] regionserver.HStore(327): Store=8cd2040dc039667d68bc1a24f5f2322f/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T01:14:25,071 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1038): replaying wal for 8cd2040dc039667d68bc1a24f5f2322f 2024-12-07T01:14:25,072 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/default/test/8cd2040dc039667d68bc1a24f5f2322f 2024-12-07T01:14:25,073 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1093): writing seq id for af843c9df2505d2eeee3c18062e9a2a6 2024-12-07T01:14:25,073 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/default/test/8cd2040dc039667d68bc1a24f5f2322f 2024-12-07T01:14:25,073 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure 73ec7ce1c93542e31d99b7b4474a9750, server=ec1863dc21e5,42915,1733534061580 because future has completed 2024-12-07T01:14:25,073 INFO [StoreOpener-1cf1ce3ee0822ff957b8ffa13f56df7d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1cf1ce3ee0822ff957b8ffa13f56df7d columnFamilyName family 2024-12-07T01:14:25,073 DEBUG [StoreOpener-1cf1ce3ee0822ff957b8ffa13f56df7d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:14:25,074 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1048): stopping wal replay for 8cd2040dc039667d68bc1a24f5f2322f 2024-12-07T01:14:25,074 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1060): Cleaning up temporary data for 8cd2040dc039667d68bc1a24f5f2322f 2024-12-07T01:14:25,074 INFO [StoreOpener-1cf1ce3ee0822ff957b8ffa13f56df7d-1 {}] regionserver.HStore(327): Store=1cf1ce3ee0822ff957b8ffa13f56df7d/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T01:14:25,074 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1038): replaying wal for 1cf1ce3ee0822ff957b8ffa13f56df7d 2024-12-07T01:14:25,075 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/default/test/1cf1ce3ee0822ff957b8ffa13f56df7d 2024-12-07T01:14:25,076 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/default/test/1cf1ce3ee0822ff957b8ffa13f56df7d 2024-12-07T01:14:25,076 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/default/test/af843c9df2505d2eeee3c18062e9a2a6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-07T01:14:25,076 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=20 2024-12-07T01:14:25,076 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure c66f402dd9e22edaa5996a3b7446808c, server=ec1863dc21e5,40693,1733534061744 in 272 msec 2024-12-07T01:14:25,076 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1048): stopping wal replay for 1cf1ce3ee0822ff957b8ffa13f56df7d 2024-12-07T01:14:25,076 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1093): writing seq id for 8cd2040dc039667d68bc1a24f5f2322f 2024-12-07T01:14:25,076 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1060): Cleaning up temporary data for 1cf1ce3ee0822ff957b8ffa13f56df7d 2024-12-07T01:14:25,077 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1114): Opened af843c9df2505d2eeee3c18062e9a2a6; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67753387, jitterRate=0.009604141116142273}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-07T01:14:25,077 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1122): Running coprocessor post-open hooks for af843c9df2505d2eeee3c18062e9a2a6 2024-12-07T01:14:25,077 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1006): Region open journal for af843c9df2505d2eeee3c18062e9a2a6: Running coprocessor pre-open hook at 1733534065062Writing region info on filesystem at 1733534065062Initializing all the Stores at 1733534065063 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534065063Cleaning up temporary data from old regions at 1733534065070 (+7 ms)Running coprocessor post-open hooks at 1733534065077 (+7 ms)Region opened successfully at 1733534065077 2024-12-07T01:14:25,078 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ddd,1733534064285.af843c9df2505d2eeee3c18062e9a2a6., pid=45, masterSystemTime=1733534064948 2024-12-07T01:14:25,079 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1093): writing seq id for 1cf1ce3ee0822ff957b8ffa13f56df7d 2024-12-07T01:14:25,079 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/default/test/8cd2040dc039667d68bc1a24f5f2322f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-07T01:14:25,080 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=c66f402dd9e22edaa5996a3b7446808c, ASSIGN in 507 msec 2024-12-07T01:14:25,080 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1114): Opened 8cd2040dc039667d68bc1a24f5f2322f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60239709, jitterRate=-0.1023583859205246}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-07T01:14:25,080 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8cd2040dc039667d68bc1a24f5f2322f 2024-12-07T01:14:25,080 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1006): Region open journal for 8cd2040dc039667d68bc1a24f5f2322f: Running coprocessor pre-open hook at 1733534065066Writing region info on filesystem at 1733534065066Initializing all the Stores at 1733534065068 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534065068Cleaning up temporary data from old regions at 1733534065074 (+6 ms)Running coprocessor post-open hooks at 1733534065080 (+6 ms)Region opened successfully at 1733534065080 2024-12-07T01:14:25,081 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ddd,1733534064285.af843c9df2505d2eeee3c18062e9a2a6. 2024-12-07T01:14:25,081 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(153): Opened test,ddd,1733534064285.af843c9df2505d2eeee3c18062e9a2a6. 2024-12-07T01:14:25,081 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(132): Open test,jjj,1733534064285.aa3dca7b6c799eb77565fb0e20ef1c15. 2024-12-07T01:14:25,082 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2236): Post open deploy tasks for test,www,1733534064285.8cd2040dc039667d68bc1a24f5f2322f., pid=36, masterSystemTime=1733534064899 2024-12-07T01:14:25,082 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7752): Opening region: {ENCODED => aa3dca7b6c799eb77565fb0e20ef1c15, NAME => 'test,jjj,1733534064285.aa3dca7b6c799eb77565fb0e20ef1c15.', STARTKEY => 'jjj', ENDKEY => 'kkk'} 2024-12-07T01:14:25,082 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test aa3dca7b6c799eb77565fb0e20ef1c15 2024-12-07T01:14:25,082 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(898): Instantiated test,jjj,1733534064285.aa3dca7b6c799eb77565fb0e20ef1c15.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T01:14:25,082 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7794): checking encryption for aa3dca7b6c799eb77565fb0e20ef1c15 2024-12-07T01:14:25,082 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7797): checking classloading for aa3dca7b6c799eb77565fb0e20ef1c15 2024-12-07T01:14:25,083 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/default/test/1cf1ce3ee0822ff957b8ffa13f56df7d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-07T01:14:25,084 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1114): Opened 1cf1ce3ee0822ff957b8ffa13f56df7d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59541992, jitterRate=-0.1127551794052124}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-07T01:14:25,084 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1cf1ce3ee0822ff957b8ffa13f56df7d 2024-12-07T01:14:25,084 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1006): Region open journal for 1cf1ce3ee0822ff957b8ffa13f56df7d: Running coprocessor pre-open hook at 1733534065069Writing region info on filesystem at 1733534065069Initializing all the Stores at 1733534065071 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534065071Cleaning up temporary data from old regions at 1733534065076 (+5 ms)Running coprocessor post-open hooks at 1733534065084 (+8 ms)Region opened successfully at 1733534065084 2024-12-07T01:14:25,084 INFO [StoreOpener-aa3dca7b6c799eb77565fb0e20ef1c15-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region aa3dca7b6c799eb77565fb0e20ef1c15 2024-12-07T01:14:25,085 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2266): Finished post open deploy task for test,www,1733534064285.8cd2040dc039667d68bc1a24f5f2322f. 2024-12-07T01:14:25,085 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(153): Opened test,www,1733534064285.8cd2040dc039667d68bc1a24f5f2322f. 2024-12-07T01:14:25,085 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2236): Post open deploy tasks for test,xxx,1733534064285.1cf1ce3ee0822ff957b8ffa13f56df7d., pid=46, masterSystemTime=1733534064990 2024-12-07T01:14:25,085 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(132): Open test,eee,1733534064285.c0a570532487ea0ebbc51cbec9ff7e37. 2024-12-07T01:14:25,086 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7752): Opening region: {ENCODED => c0a570532487ea0ebbc51cbec9ff7e37, NAME => 'test,eee,1733534064285.c0a570532487ea0ebbc51cbec9ff7e37.', STARTKEY => 'eee', ENDKEY => 'fff'} 2024-12-07T01:14:25,086 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-12-07T01:14:25,086 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; OpenRegionProcedure 73ec7ce1c93542e31d99b7b4474a9750, server=ec1863dc21e5,42915,1733534061580 in 338 msec 2024-12-07T01:14:25,086 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test c0a570532487ea0ebbc51cbec9ff7e37 2024-12-07T01:14:25,086 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(898): Instantiated test,eee,1733534064285.c0a570532487ea0ebbc51cbec9ff7e37.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T01:14:25,086 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7794): checking encryption for c0a570532487ea0ebbc51cbec9ff7e37 2024-12-07T01:14:25,086 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7797): checking classloading for c0a570532487ea0ebbc51cbec9ff7e37 2024-12-07T01:14:25,087 INFO [StoreOpener-aa3dca7b6c799eb77565fb0e20ef1c15-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region aa3dca7b6c799eb77565fb0e20ef1c15 columnFamilyName family 2024-12-07T01:14:25,087 DEBUG [StoreOpener-aa3dca7b6c799eb77565fb0e20ef1c15-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:14:25,088 INFO [StoreOpener-aa3dca7b6c799eb77565fb0e20ef1c15-1 {}] regionserver.HStore(327): Store=aa3dca7b6c799eb77565fb0e20ef1c15/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T01:14:25,088 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2266): Finished post open deploy task for test,xxx,1733534064285.1cf1ce3ee0822ff957b8ffa13f56df7d. 2024-12-07T01:14:25,088 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(153): Opened test,xxx,1733534064285.1cf1ce3ee0822ff957b8ffa13f56df7d. 2024-12-07T01:14:25,088 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(132): Open test,lll,1733534064285.d2b1eb41dda894f4f7b826a2826a9c7b. 2024-12-07T01:14:25,088 INFO [StoreOpener-c0a570532487ea0ebbc51cbec9ff7e37-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region c0a570532487ea0ebbc51cbec9ff7e37 2024-12-07T01:14:25,088 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1038): replaying wal for aa3dca7b6c799eb77565fb0e20ef1c15 2024-12-07T01:14:25,088 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7752): Opening region: {ENCODED => d2b1eb41dda894f4f7b826a2826a9c7b, NAME => 'test,lll,1733534064285.d2b1eb41dda894f4f7b826a2826a9c7b.', STARTKEY => 'lll', ENDKEY => 'mmm'} 2024-12-07T01:14:25,089 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test d2b1eb41dda894f4f7b826a2826a9c7b 2024-12-07T01:14:25,089 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(898): Instantiated test,lll,1733534064285.d2b1eb41dda894f4f7b826a2826a9c7b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T01:14:25,089 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7794): checking encryption for d2b1eb41dda894f4f7b826a2826a9c7b 2024-12-07T01:14:25,089 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7797): checking classloading for d2b1eb41dda894f4f7b826a2826a9c7b 2024-12-07T01:14:25,089 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=73ec7ce1c93542e31d99b7b4474a9750, ASSIGN in 517 msec 2024-12-07T01:14:25,090 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/default/test/aa3dca7b6c799eb77565fb0e20ef1c15 2024-12-07T01:14:25,090 INFO [StoreOpener-c0a570532487ea0ebbc51cbec9ff7e37-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c0a570532487ea0ebbc51cbec9ff7e37 columnFamilyName family 2024-12-07T01:14:25,090 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/default/test/aa3dca7b6c799eb77565fb0e20ef1c15 2024-12-07T01:14:25,090 DEBUG [StoreOpener-c0a570532487ea0ebbc51cbec9ff7e37-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:14:25,091 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1048): stopping wal replay for aa3dca7b6c799eb77565fb0e20ef1c15 2024-12-07T01:14:25,091 INFO [StoreOpener-c0a570532487ea0ebbc51cbec9ff7e37-1 {}] regionserver.HStore(327): Store=c0a570532487ea0ebbc51cbec9ff7e37/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T01:14:25,091 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1060): Cleaning up temporary data for aa3dca7b6c799eb77565fb0e20ef1c15 2024-12-07T01:14:25,091 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1038): replaying wal for c0a570532487ea0ebbc51cbec9ff7e37 2024-12-07T01:14:25,091 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=1cf1ce3ee0822ff957b8ffa13f56df7d, regionState=OPEN, openSeqNum=2, regionLocation=ec1863dc21e5,36079,1733534061701 2024-12-07T01:14:25,091 INFO [StoreOpener-d2b1eb41dda894f4f7b826a2826a9c7b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region d2b1eb41dda894f4f7b826a2826a9c7b 2024-12-07T01:14:25,092 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/default/test/c0a570532487ea0ebbc51cbec9ff7e37 2024-12-07T01:14:25,093 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/default/test/c0a570532487ea0ebbc51cbec9ff7e37 2024-12-07T01:14:25,094 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1093): writing seq id for aa3dca7b6c799eb77565fb0e20ef1c15 2024-12-07T01:14:25,094 INFO [StoreOpener-d2b1eb41dda894f4f7b826a2826a9c7b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d2b1eb41dda894f4f7b826a2826a9c7b columnFamilyName family 2024-12-07T01:14:25,094 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1048): stopping wal replay for c0a570532487ea0ebbc51cbec9ff7e37 2024-12-07T01:14:25,094 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1060): Cleaning up temporary data for c0a570532487ea0ebbc51cbec9ff7e37 2024-12-07T01:14:25,094 DEBUG [StoreOpener-d2b1eb41dda894f4f7b826a2826a9c7b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:14:25,094 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=46, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1cf1ce3ee0822ff957b8ffa13f56df7d, server=ec1863dc21e5,36079,1733534061701 because future has completed 2024-12-07T01:14:25,094 INFO [StoreOpener-d2b1eb41dda894f4f7b826a2826a9c7b-1 {}] regionserver.HStore(327): Store=d2b1eb41dda894f4f7b826a2826a9c7b/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T01:14:25,095 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=8cd2040dc039667d68bc1a24f5f2322f, regionState=OPEN, openSeqNum=2, regionLocation=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:25,095 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1038): replaying wal for d2b1eb41dda894f4f7b826a2826a9c7b 2024-12-07T01:14:25,095 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/default/test/d2b1eb41dda894f4f7b826a2826a9c7b 2024-12-07T01:14:25,096 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/default/test/d2b1eb41dda894f4f7b826a2826a9c7b 2024-12-07T01:14:25,096 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1093): writing seq id for c0a570532487ea0ebbc51cbec9ff7e37 2024-12-07T01:14:25,097 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1048): stopping wal replay for d2b1eb41dda894f4f7b826a2826a9c7b 2024-12-07T01:14:25,097 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1060): Cleaning up temporary data for d2b1eb41dda894f4f7b826a2826a9c7b 2024-12-07T01:14:25,098 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/default/test/aa3dca7b6c799eb77565fb0e20ef1c15/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-07T01:14:25,098 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8cd2040dc039667d68bc1a24f5f2322f, server=ec1863dc21e5,42915,1733534061580 because future has completed 2024-12-07T01:14:25,098 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1114): Opened aa3dca7b6c799eb77565fb0e20ef1c15; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74079745, jitterRate=0.10387422144412994}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-07T01:14:25,099 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1122): Running coprocessor post-open hooks for aa3dca7b6c799eb77565fb0e20ef1c15 2024-12-07T01:14:25,099 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1006): Region open journal for aa3dca7b6c799eb77565fb0e20ef1c15: Running coprocessor pre-open hook at 1733534065083Writing region info on filesystem at 1733534065083Initializing all the Stores at 1733534065084 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534065084Cleaning up temporary data from old regions at 1733534065091 (+7 ms)Running coprocessor post-open hooks at 1733534065099 (+8 ms)Region opened successfully at 1733534065099 2024-12-07T01:14:25,100 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2236): Post open deploy tasks for test,jjj,1733534064285.aa3dca7b6c799eb77565fb0e20ef1c15., pid=49, masterSystemTime=1733534064948 2024-12-07T01:14:25,101 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1093): writing seq id for d2b1eb41dda894f4f7b826a2826a9c7b 2024-12-07T01:14:25,103 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=46, resume processing ppid=28 2024-12-07T01:14:25,103 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, ppid=28, state=SUCCESS, hasLock=false; OpenRegionProcedure 1cf1ce3ee0822ff957b8ffa13f56df7d, server=ec1863dc21e5,36079,1733534061701 in 260 msec 2024-12-07T01:14:25,103 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2266): Finished post open deploy task for test,jjj,1733534064285.aa3dca7b6c799eb77565fb0e20ef1c15. 2024-12-07T01:14:25,104 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(153): Opened test,jjj,1733534064285.aa3dca7b6c799eb77565fb0e20ef1c15. 2024-12-07T01:14:25,104 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(132): Open test,,1733534064285.976bbe8d7afc3b1e02af11d1b515e82c. 2024-12-07T01:14:25,104 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7752): Opening region: {ENCODED => 976bbe8d7afc3b1e02af11d1b515e82c, NAME => 'test,,1733534064285.976bbe8d7afc3b1e02af11d1b515e82c.', STARTKEY => '', ENDKEY => 'bbb'} 2024-12-07T01:14:25,104 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 976bbe8d7afc3b1e02af11d1b515e82c 2024-12-07T01:14:25,104 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(898): Instantiated test,,1733534064285.976bbe8d7afc3b1e02af11d1b515e82c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T01:14:25,104 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7794): checking encryption for 976bbe8d7afc3b1e02af11d1b515e82c 2024-12-07T01:14:25,104 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7797): checking classloading for 976bbe8d7afc3b1e02af11d1b515e82c 2024-12-07T01:14:25,105 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=aa3dca7b6c799eb77565fb0e20ef1c15, regionState=OPEN, openSeqNum=2, regionLocation=ec1863dc21e5,40693,1733534061744 2024-12-07T01:14:25,107 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=49, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure aa3dca7b6c799eb77565fb0e20ef1c15, server=ec1863dc21e5,40693,1733534061744 because future has completed 2024-12-07T01:14:25,109 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=1cf1ce3ee0822ff957b8ffa13f56df7d, ASSIGN in 537 msec 2024-12-07T01:14:25,113 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=14 2024-12-07T01:14:25,113 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=14, state=SUCCESS, hasLock=false; OpenRegionProcedure aa3dca7b6c799eb77565fb0e20ef1c15, server=ec1863dc21e5,40693,1733534061744 in 258 msec 2024-12-07T01:14:25,113 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/default/test/c0a570532487ea0ebbc51cbec9ff7e37/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-07T01:14:25,113 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/default/test/d2b1eb41dda894f4f7b826a2826a9c7b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-07T01:14:25,114 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1114): Opened c0a570532487ea0ebbc51cbec9ff7e37; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59457962, jitterRate=-0.11400732398033142}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-07T01:14:25,114 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1114): Opened d2b1eb41dda894f4f7b826a2826a9c7b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65571915, jitterRate=-0.02290232479572296}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-07T01:14:25,114 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c0a570532487ea0ebbc51cbec9ff7e37 2024-12-07T01:14:25,114 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d2b1eb41dda894f4f7b826a2826a9c7b 2024-12-07T01:14:25,114 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1006): Region open journal for c0a570532487ea0ebbc51cbec9ff7e37: Running coprocessor pre-open hook at 1733534065086Writing region info on filesystem at 1733534065086Initializing all the Stores at 1733534065088 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534065088Cleaning up temporary data from old regions at 1733534065094 (+6 ms)Running coprocessor post-open hooks at 1733534065114 (+20 ms)Region opened successfully at 1733534065114 2024-12-07T01:14:25,114 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1006): Region open journal for d2b1eb41dda894f4f7b826a2826a9c7b: Running coprocessor pre-open hook at 1733534065089Writing region info on filesystem at 1733534065089Initializing all the Stores at 1733534065091 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534065091Cleaning up temporary data from old regions at 1733534065097 (+6 ms)Running coprocessor post-open hooks at 1733534065114 (+17 ms)Region opened successfully at 1733534065114 2024-12-07T01:14:25,115 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2236): Post open deploy tasks for test,eee,1733534064285.c0a570532487ea0ebbc51cbec9ff7e37., pid=38, masterSystemTime=1733534064899 2024-12-07T01:14:25,115 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2236): Post open deploy tasks for test,lll,1733534064285.d2b1eb41dda894f4f7b826a2826a9c7b., pid=51, masterSystemTime=1733534064990 2024-12-07T01:14:25,116 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=aa3dca7b6c799eb77565fb0e20ef1c15, ASSIGN in 544 msec 2024-12-07T01:14:25,117 INFO [StoreOpener-976bbe8d7afc3b1e02af11d1b515e82c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 976bbe8d7afc3b1e02af11d1b515e82c 2024-12-07T01:14:25,118 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2266): Finished post open deploy task for test,lll,1733534064285.d2b1eb41dda894f4f7b826a2826a9c7b. 2024-12-07T01:14:25,118 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(153): Opened test,lll,1733534064285.d2b1eb41dda894f4f7b826a2826a9c7b. 2024-12-07T01:14:25,118 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(132): Open test,ooo,1733534064285.324bdaae20264b3880845c3082304e57. 2024-12-07T01:14:25,119 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7752): Opening region: {ENCODED => 324bdaae20264b3880845c3082304e57, NAME => 'test,ooo,1733534064285.324bdaae20264b3880845c3082304e57.', STARTKEY => 'ooo', ENDKEY => 'ppp'} 2024-12-07T01:14:25,119 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 324bdaae20264b3880845c3082304e57 2024-12-07T01:14:25,119 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(898): Instantiated test,ooo,1733534064285.324bdaae20264b3880845c3082304e57.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T01:14:25,119 INFO [StoreOpener-976bbe8d7afc3b1e02af11d1b515e82c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 976bbe8d7afc3b1e02af11d1b515e82c columnFamilyName family 2024-12-07T01:14:25,119 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7794): checking encryption for 324bdaae20264b3880845c3082304e57 2024-12-07T01:14:25,119 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7797): checking classloading for 324bdaae20264b3880845c3082304e57 2024-12-07T01:14:25,119 DEBUG [StoreOpener-976bbe8d7afc3b1e02af11d1b515e82c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:14:25,120 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2266): Finished post open deploy task for test,eee,1733534064285.c0a570532487ea0ebbc51cbec9ff7e37. 2024-12-07T01:14:25,120 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(153): Opened test,eee,1733534064285.c0a570532487ea0ebbc51cbec9ff7e37. 2024-12-07T01:14:25,120 INFO [StoreOpener-976bbe8d7afc3b1e02af11d1b515e82c-1 {}] regionserver.HStore(327): Store=976bbe8d7afc3b1e02af11d1b515e82c/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T01:14:25,120 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1038): replaying wal for 976bbe8d7afc3b1e02af11d1b515e82c 2024-12-07T01:14:25,122 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/default/test/976bbe8d7afc3b1e02af11d1b515e82c 2024-12-07T01:14:25,122 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/default/test/976bbe8d7afc3b1e02af11d1b515e82c 2024-12-07T01:14:25,123 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=27 2024-12-07T01:14:25,123 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=27, state=SUCCESS, hasLock=false; OpenRegionProcedure 8cd2040dc039667d68bc1a24f5f2322f, server=ec1863dc21e5,42915,1733534061580 in 345 msec 2024-12-07T01:14:25,123 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1048): stopping wal replay for 976bbe8d7afc3b1e02af11d1b515e82c 2024-12-07T01:14:25,123 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1060): Cleaning up temporary data for 976bbe8d7afc3b1e02af11d1b515e82c 2024-12-07T01:14:25,125 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=8cd2040dc039667d68bc1a24f5f2322f, ASSIGN in 554 msec 2024-12-07T01:14:25,126 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1093): writing seq id for 976bbe8d7afc3b1e02af11d1b515e82c 2024-12-07T01:14:25,127 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=c0a570532487ea0ebbc51cbec9ff7e37, regionState=OPEN, openSeqNum=2, regionLocation=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:25,129 INFO [StoreOpener-324bdaae20264b3880845c3082304e57-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 324bdaae20264b3880845c3082304e57 2024-12-07T01:14:25,130 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/default/test/976bbe8d7afc3b1e02af11d1b515e82c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-07T01:14:25,131 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure c0a570532487ea0ebbc51cbec9ff7e37, server=ec1863dc21e5,42915,1733534061580 because future has completed 2024-12-07T01:14:25,131 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1114): Opened 976bbe8d7afc3b1e02af11d1b515e82c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60051142, jitterRate=-0.10516825318336487}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-07T01:14:25,131 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 976bbe8d7afc3b1e02af11d1b515e82c 2024-12-07T01:14:25,131 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1006): Region open journal for 976bbe8d7afc3b1e02af11d1b515e82c: Running coprocessor pre-open hook at 1733534065105Writing region info on filesystem at 1733534065105Initializing all the Stores at 1733534065106 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534065106Cleaning up temporary data from old regions at 1733534065123 (+17 ms)Running coprocessor post-open hooks at 1733534065131 (+8 ms)Region opened successfully at 1733534065131 2024-12-07T01:14:25,131 INFO [StoreOpener-324bdaae20264b3880845c3082304e57-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 324bdaae20264b3880845c3082304e57 columnFamilyName family 2024-12-07T01:14:25,131 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=d2b1eb41dda894f4f7b826a2826a9c7b, regionState=OPEN, openSeqNum=2, regionLocation=ec1863dc21e5,36079,1733534061701 2024-12-07T01:14:25,131 DEBUG [StoreOpener-324bdaae20264b3880845c3082304e57-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:14:25,133 INFO [StoreOpener-324bdaae20264b3880845c3082304e57-1 {}] regionserver.HStore(327): Store=324bdaae20264b3880845c3082304e57/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T01:14:25,133 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1038): replaying wal for 324bdaae20264b3880845c3082304e57 2024-12-07T01:14:25,134 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2236): Post open deploy tasks for test,,1733534064285.976bbe8d7afc3b1e02af11d1b515e82c., pid=47, masterSystemTime=1733534064948 2024-12-07T01:14:25,135 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/default/test/324bdaae20264b3880845c3082304e57 2024-12-07T01:14:25,135 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/default/test/324bdaae20264b3880845c3082304e57 2024-12-07T01:14:25,136 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=51, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure d2b1eb41dda894f4f7b826a2826a9c7b, server=ec1863dc21e5,36079,1733534061701 because future has completed 2024-12-07T01:14:25,136 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1048): stopping wal replay for 324bdaae20264b3880845c3082304e57 2024-12-07T01:14:25,136 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1060): Cleaning up temporary data for 324bdaae20264b3880845c3082304e57 2024-12-07T01:14:25,137 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2266): Finished post open deploy task for test,,1733534064285.976bbe8d7afc3b1e02af11d1b515e82c. 2024-12-07T01:14:25,137 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(153): Opened test,,1733534064285.976bbe8d7afc3b1e02af11d1b515e82c. 2024-12-07T01:14:25,137 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(132): Open test,yyy,1733534064285.b59f40eb016096571b13ae445f262449. 2024-12-07T01:14:25,137 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7752): Opening region: {ENCODED => b59f40eb016096571b13ae445f262449, NAME => 'test,yyy,1733534064285.b59f40eb016096571b13ae445f262449.', STARTKEY => 'yyy', ENDKEY => 'zzz'} 2024-12-07T01:14:25,138 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test b59f40eb016096571b13ae445f262449 2024-12-07T01:14:25,138 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(898): Instantiated test,yyy,1733534064285.b59f40eb016096571b13ae445f262449.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T01:14:25,138 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7794): checking encryption for b59f40eb016096571b13ae445f262449 2024-12-07T01:14:25,138 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7797): checking classloading for b59f40eb016096571b13ae445f262449 2024-12-07T01:14:25,138 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1093): writing seq id for 324bdaae20264b3880845c3082304e57 2024-12-07T01:14:25,139 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=9 2024-12-07T01:14:25,139 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=9, state=SUCCESS, hasLock=false; OpenRegionProcedure c0a570532487ea0ebbc51cbec9ff7e37, server=ec1863dc21e5,42915,1733534061580 in 347 msec 2024-12-07T01:14:25,140 INFO [StoreOpener-b59f40eb016096571b13ae445f262449-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region b59f40eb016096571b13ae445f262449 2024-12-07T01:14:25,141 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/default/test/324bdaae20264b3880845c3082304e57/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-07T01:14:25,142 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1114): Opened 324bdaae20264b3880845c3082304e57; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66108685, jitterRate=-0.014903828501701355}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-07T01:14:25,142 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 324bdaae20264b3880845c3082304e57 2024-12-07T01:14:25,142 INFO [StoreOpener-b59f40eb016096571b13ae445f262449-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b59f40eb016096571b13ae445f262449 columnFamilyName family 2024-12-07T01:14:25,142 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=c0a570532487ea0ebbc51cbec9ff7e37, ASSIGN in 570 msec 2024-12-07T01:14:25,142 DEBUG [StoreOpener-b59f40eb016096571b13ae445f262449-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:14:25,142 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1006): Region open journal for 324bdaae20264b3880845c3082304e57: Running coprocessor pre-open hook at 1733534065119Writing region info on filesystem at 1733534065119Initializing all the Stores at 1733534065121 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534065121Cleaning up temporary data from old regions at 1733534065136 (+15 ms)Running coprocessor post-open hooks at 1733534065142 (+6 ms)Region opened successfully at 1733534065142 2024-12-07T01:14:25,143 INFO [StoreOpener-b59f40eb016096571b13ae445f262449-1 {}] regionserver.HStore(327): Store=b59f40eb016096571b13ae445f262449/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T01:14:25,143 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1038): replaying wal for b59f40eb016096571b13ae445f262449 2024-12-07T01:14:25,143 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ooo,1733534064285.324bdaae20264b3880845c3082304e57., pid=55, masterSystemTime=1733534064990 2024-12-07T01:14:25,143 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=976bbe8d7afc3b1e02af11d1b515e82c, regionState=OPEN, openSeqNum=2, regionLocation=ec1863dc21e5,40693,1733534061744 2024-12-07T01:14:25,144 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/default/test/b59f40eb016096571b13ae445f262449 2024-12-07T01:14:25,144 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/default/test/b59f40eb016096571b13ae445f262449 2024-12-07T01:14:25,144 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1048): stopping wal replay for b59f40eb016096571b13ae445f262449 2024-12-07T01:14:25,145 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1060): Cleaning up temporary data for b59f40eb016096571b13ae445f262449 2024-12-07T01:14:25,146 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ooo,1733534064285.324bdaae20264b3880845c3082304e57. 2024-12-07T01:14:25,146 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(153): Opened test,ooo,1733534064285.324bdaae20264b3880845c3082304e57. 2024-12-07T01:14:25,146 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(132): Open test,rrr,1733534064285.fa4e20ed49e59ee4cf63b6d84515872f. 2024-12-07T01:14:25,146 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7752): Opening region: {ENCODED => fa4e20ed49e59ee4cf63b6d84515872f, NAME => 'test,rrr,1733534064285.fa4e20ed49e59ee4cf63b6d84515872f.', STARTKEY => 'rrr', ENDKEY => 'sss'} 2024-12-07T01:14:25,146 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=47, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 976bbe8d7afc3b1e02af11d1b515e82c, server=ec1863dc21e5,40693,1733534061744 because future has completed 2024-12-07T01:14:25,146 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test fa4e20ed49e59ee4cf63b6d84515872f 2024-12-07T01:14:25,147 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(898): Instantiated test,rrr,1733534064285.fa4e20ed49e59ee4cf63b6d84515872f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T01:14:25,147 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7794): checking encryption for fa4e20ed49e59ee4cf63b6d84515872f 2024-12-07T01:14:25,147 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1093): writing seq id for b59f40eb016096571b13ae445f262449 2024-12-07T01:14:25,147 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7797): checking classloading for fa4e20ed49e59ee4cf63b6d84515872f 2024-12-07T01:14:25,147 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=324bdaae20264b3880845c3082304e57, regionState=OPEN, openSeqNum=2, regionLocation=ec1863dc21e5,36079,1733534061701 2024-12-07T01:14:25,149 INFO [StoreOpener-fa4e20ed49e59ee4cf63b6d84515872f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region fa4e20ed49e59ee4cf63b6d84515872f 2024-12-07T01:14:25,150 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/default/test/b59f40eb016096571b13ae445f262449/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-07T01:14:25,151 INFO [StoreOpener-fa4e20ed49e59ee4cf63b6d84515872f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fa4e20ed49e59ee4cf63b6d84515872f columnFamilyName family 2024-12-07T01:14:25,151 DEBUG [StoreOpener-fa4e20ed49e59ee4cf63b6d84515872f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:14:25,151 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1114): Opened b59f40eb016096571b13ae445f262449; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58866044, jitterRate=-0.12282758951187134}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-07T01:14:25,151 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b59f40eb016096571b13ae445f262449 2024-12-07T01:14:25,151 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1006): Region open journal for b59f40eb016096571b13ae445f262449: Running coprocessor pre-open hook at 1733534065138Writing region info on filesystem at 1733534065138Initializing all the Stores at 1733534065139 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534065140 (+1 ms)Cleaning up temporary data from old regions at 1733534065145 (+5 ms)Running coprocessor post-open hooks at 1733534065151 (+6 ms)Region opened successfully at 1733534065151 2024-12-07T01:14:25,152 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=55, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure 324bdaae20264b3880845c3082304e57, server=ec1863dc21e5,36079,1733534061701 because future has completed 2024-12-07T01:14:25,152 INFO [StoreOpener-fa4e20ed49e59ee4cf63b6d84515872f-1 {}] regionserver.HStore(327): Store=fa4e20ed49e59ee4cf63b6d84515872f/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T01:14:25,153 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1038): replaying wal for fa4e20ed49e59ee4cf63b6d84515872f 2024-12-07T01:14:25,153 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2236): Post open deploy tasks for test,yyy,1733534064285.b59f40eb016096571b13ae445f262449., pid=39, masterSystemTime=1733534064948 2024-12-07T01:14:25,154 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/default/test/fa4e20ed49e59ee4cf63b6d84515872f 2024-12-07T01:14:25,154 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/default/test/fa4e20ed49e59ee4cf63b6d84515872f 2024-12-07T01:14:25,155 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1048): stopping wal replay for fa4e20ed49e59ee4cf63b6d84515872f 2024-12-07T01:14:25,155 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1060): Cleaning up temporary data for fa4e20ed49e59ee4cf63b6d84515872f 2024-12-07T01:14:25,157 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1093): writing seq id for fa4e20ed49e59ee4cf63b6d84515872f 2024-12-07T01:14:25,160 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/default/test/fa4e20ed49e59ee4cf63b6d84515872f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-07T01:14:25,160 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2266): Finished post open deploy task for test,yyy,1733534064285.b59f40eb016096571b13ae445f262449. 2024-12-07T01:14:25,160 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(153): Opened test,yyy,1733534064285.b59f40eb016096571b13ae445f262449. 2024-12-07T01:14:25,161 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1114): Opened fa4e20ed49e59ee4cf63b6d84515872f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65949378, jitterRate=-0.017277687788009644}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-07T01:14:25,161 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fa4e20ed49e59ee4cf63b6d84515872f 2024-12-07T01:14:25,161 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=5 2024-12-07T01:14:25,161 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1006): Region open journal for fa4e20ed49e59ee4cf63b6d84515872f: Running coprocessor pre-open hook at 1733534065147Writing region info on filesystem at 1733534065147Initializing all the Stores at 1733534065148 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534065148Cleaning up temporary data from old regions at 1733534065155 (+7 ms)Running coprocessor post-open hooks at 1733534065161 (+6 ms)Region opened successfully at 1733534065161 2024-12-07T01:14:25,161 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 976bbe8d7afc3b1e02af11d1b515e82c, server=ec1863dc21e5,40693,1733534061744 in 313 msec 2024-12-07T01:14:25,162 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2236): Post open deploy tasks for test,rrr,1733534064285.fa4e20ed49e59ee4cf63b6d84515872f., pid=56, masterSystemTime=1733534064990 2024-12-07T01:14:25,164 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=976bbe8d7afc3b1e02af11d1b515e82c, ASSIGN in 592 msec 2024-12-07T01:14:25,164 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2266): Finished post open deploy task for test,rrr,1733534064285.fa4e20ed49e59ee4cf63b6d84515872f. 2024-12-07T01:14:25,164 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(153): Opened test,rrr,1733534064285.fa4e20ed49e59ee4cf63b6d84515872f. 2024-12-07T01:14:25,164 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(132): Open test,fff,1733534064285.37d131a6ac713be02dc73a29079210f5. 2024-12-07T01:14:25,164 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46141 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=ec1863dc21e5,36079,1733534061701, table=test, region=d2b1eb41dda894f4f7b826a2826a9c7b. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-07T01:14:25,165 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46141 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=ec1863dc21e5,36079,1733534061701, table=test, region=324bdaae20264b3880845c3082304e57. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-07T01:14:25,165 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7752): Opening region: {ENCODED => 37d131a6ac713be02dc73a29079210f5, NAME => 'test,fff,1733534064285.37d131a6ac713be02dc73a29079210f5.', STARTKEY => 'fff', ENDKEY => 'ggg'} 2024-12-07T01:14:25,165 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 37d131a6ac713be02dc73a29079210f5 2024-12-07T01:14:25,165 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(898): Instantiated test,fff,1733534064285.37d131a6ac713be02dc73a29079210f5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T01:14:25,165 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7794): checking encryption for 37d131a6ac713be02dc73a29079210f5 2024-12-07T01:14:25,165 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7797): checking classloading for 37d131a6ac713be02dc73a29079210f5 2024-12-07T01:14:25,165 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=fa4e20ed49e59ee4cf63b6d84515872f, regionState=OPEN, openSeqNum=2, regionLocation=ec1863dc21e5,36079,1733534061701 2024-12-07T01:14:25,167 INFO [StoreOpener-37d131a6ac713be02dc73a29079210f5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 37d131a6ac713be02dc73a29079210f5 2024-12-07T01:14:25,168 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure fa4e20ed49e59ee4cf63b6d84515872f, server=ec1863dc21e5,36079,1733534061701 because future has completed 2024-12-07T01:14:25,169 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=b59f40eb016096571b13ae445f262449, regionState=OPEN, openSeqNum=2, regionLocation=ec1863dc21e5,40693,1733534061744 2024-12-07T01:14:25,169 INFO [StoreOpener-37d131a6ac713be02dc73a29079210f5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 37d131a6ac713be02dc73a29079210f5 columnFamilyName family 2024-12-07T01:14:25,169 DEBUG [StoreOpener-37d131a6ac713be02dc73a29079210f5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:14:25,170 INFO [StoreOpener-37d131a6ac713be02dc73a29079210f5-1 {}] regionserver.HStore(327): Store=37d131a6ac713be02dc73a29079210f5/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T01:14:25,170 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1038): replaying wal for 37d131a6ac713be02dc73a29079210f5 2024-12-07T01:14:25,171 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/default/test/37d131a6ac713be02dc73a29079210f5 2024-12-07T01:14:25,171 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/default/test/37d131a6ac713be02dc73a29079210f5 2024-12-07T01:14:25,172 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure b59f40eb016096571b13ae445f262449, server=ec1863dc21e5,40693,1733534061744 because future has completed 2024-12-07T01:14:25,172 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1048): stopping wal replay for 37d131a6ac713be02dc73a29079210f5 2024-12-07T01:14:25,172 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1060): Cleaning up temporary data for 37d131a6ac713be02dc73a29079210f5 2024-12-07T01:14:25,174 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1093): writing seq id for 37d131a6ac713be02dc73a29079210f5 2024-12-07T01:14:25,175 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=22 2024-12-07T01:14:25,175 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=22, state=SUCCESS, hasLock=false; OpenRegionProcedure fa4e20ed49e59ee4cf63b6d84515872f, server=ec1863dc21e5,36079,1733534061701 in 288 msec 2024-12-07T01:14:25,176 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/default/test/37d131a6ac713be02dc73a29079210f5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-07T01:14:25,177 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1114): Opened 37d131a6ac713be02dc73a29079210f5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58889253, jitterRate=-0.12248174846172333}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-07T01:14:25,177 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 37d131a6ac713be02dc73a29079210f5 2024-12-07T01:14:25,177 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1006): Region open journal for 37d131a6ac713be02dc73a29079210f5: Running coprocessor pre-open hook at 1733534065165Writing region info on filesystem at 1733534065165Initializing all the Stores at 1733534065167 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534065167Cleaning up temporary data from old regions at 1733534065172 (+5 ms)Running coprocessor post-open hooks at 1733534065177 (+5 ms)Region opened successfully at 1733534065177 2024-12-07T01:14:25,178 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=fa4e20ed49e59ee4cf63b6d84515872f, ASSIGN in 607 msec 2024-12-07T01:14:25,178 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2236): Post open deploy tasks for test,fff,1733534064285.37d131a6ac713be02dc73a29079210f5., pid=53, masterSystemTime=1733534064990 2024-12-07T01:14:25,180 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2266): Finished post open deploy task for test,fff,1733534064285.37d131a6ac713be02dc73a29079210f5. 2024-12-07T01:14:25,180 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(153): Opened test,fff,1733534064285.37d131a6ac713be02dc73a29079210f5. 2024-12-07T01:14:25,182 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=29 2024-12-07T01:14:25,182 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure b59f40eb016096571b13ae445f262449, server=ec1863dc21e5,40693,1733534061744 in 384 msec 2024-12-07T01:14:25,183 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=b59f40eb016096571b13ae445f262449, ASSIGN in 613 msec 2024-12-07T01:14:25,184 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=37d131a6ac713be02dc73a29079210f5, regionState=OPEN, openSeqNum=2, regionLocation=ec1863dc21e5,36079,1733534061701 2024-12-07T01:14:25,187 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=53, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 37d131a6ac713be02dc73a29079210f5, server=ec1863dc21e5,36079,1733534061701 because future has completed 2024-12-07T01:14:25,189 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=19 2024-12-07T01:14:25,189 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=19, state=SUCCESS, hasLock=false; OpenRegionProcedure 324bdaae20264b3880845c3082304e57, server=ec1863dc21e5,36079,1733534061701 in 305 msec 2024-12-07T01:14:25,191 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=324bdaae20264b3880845c3082304e57, ASSIGN in 620 msec 2024-12-07T01:14:25,194 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=10 2024-12-07T01:14:25,194 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 37d131a6ac713be02dc73a29079210f5, server=ec1863dc21e5,36079,1733534061701 in 318 msec 2024-12-07T01:14:25,196 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=37d131a6ac713be02dc73a29079210f5, ASSIGN in 625 msec 2024-12-07T01:14:25,199 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=16 2024-12-07T01:14:25,199 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure d2b1eb41dda894f4f7b826a2826a9c7b, server=ec1863dc21e5,36079,1733534061701 in 333 msec 2024-12-07T01:14:25,201 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=d2b1eb41dda894f4f7b826a2826a9c7b, ASSIGN in 630 msec 2024-12-07T01:14:25,202 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=af843c9df2505d2eeee3c18062e9a2a6, regionState=OPEN, openSeqNum=2, regionLocation=ec1863dc21e5,40693,1733534061744 2024-12-07T01:14:25,204 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=45, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure af843c9df2505d2eeee3c18062e9a2a6, server=ec1863dc21e5,40693,1733534061744 because future has completed 2024-12-07T01:14:25,205 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=6d8689bfa03ba7aed23b15e5ff5d23ac, regionState=OPEN, openSeqNum=2, regionLocation=ec1863dc21e5,36079,1733534061701 2024-12-07T01:14:25,207 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=52, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6d8689bfa03ba7aed23b15e5ff5d23ac, server=ec1863dc21e5,36079,1733534061701 because future has completed 2024-12-07T01:14:25,209 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=8 2024-12-07T01:14:25,209 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure af843c9df2505d2eeee3c18062e9a2a6, server=ec1863dc21e5,40693,1733534061744 in 373 msec 2024-12-07T01:14:25,210 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=af843c9df2505d2eeee3c18062e9a2a6, ASSIGN in 640 msec 2024-12-07T01:14:25,214 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=52, resume processing ppid=7 2024-12-07T01:14:25,214 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 6d8689bfa03ba7aed23b15e5ff5d23ac, server=ec1863dc21e5,36079,1733534061701 in 342 msec 2024-12-07T01:14:25,215 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=6d8689bfa03ba7aed23b15e5ff5d23ac, ASSIGN in 645 msec 2024-12-07T01:14:25,216 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=c0f43810347933724c9e6319b11ae5cc, regionState=OPEN, openSeqNum=2, regionLocation=ec1863dc21e5,40693,1733534061744 2024-12-07T01:14:25,219 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=3f148a9f8c5bea63f4744afb36553ef7, regionState=OPEN, openSeqNum=2, regionLocation=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:25,219 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=48, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure c0f43810347933724c9e6319b11ae5cc, server=ec1863dc21e5,40693,1733534061744 because future has completed 2024-12-07T01:14:25,221 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3f148a9f8c5bea63f4744afb36553ef7, server=ec1863dc21e5,42915,1733534061580 because future has completed 2024-12-07T01:14:25,223 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=48, resume processing ppid=11 2024-12-07T01:14:25,224 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure c0f43810347933724c9e6319b11ae5cc, server=ec1863dc21e5,40693,1733534061744 in 374 msec 2024-12-07T01:14:25,225 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=c0f43810347933724c9e6319b11ae5cc, ASSIGN in 655 msec 2024-12-07T01:14:25,228 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=24 2024-12-07T01:14:25,228 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=24, state=SUCCESS, hasLock=false; OpenRegionProcedure 3f148a9f8c5bea63f4744afb36553ef7, server=ec1863dc21e5,42915,1733534061580 in 457 msec 2024-12-07T01:14:25,230 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=3f148a9f8c5bea63f4744afb36553ef7, ASSIGN in 659 msec 2024-12-07T01:14:25,231 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=75c927364905f2c6bffff16328d8aaf5, regionState=OPEN, openSeqNum=2, regionLocation=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:25,233 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure 75c927364905f2c6bffff16328d8aaf5, server=ec1863dc21e5,42915,1733534061580 because future has completed 2024-12-07T01:14:25,237 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=15 2024-12-07T01:14:25,237 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=15, state=SUCCESS, hasLock=false; OpenRegionProcedure 75c927364905f2c6bffff16328d8aaf5, server=ec1863dc21e5,42915,1733534061580 in 475 msec 2024-12-07T01:14:25,240 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=4 2024-12-07T01:14:25,240 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=75c927364905f2c6bffff16328d8aaf5, ASSIGN in 669 msec 2024-12-07T01:14:25,241 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-07T01:14:25,242 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733534065241"}]},"ts":"1733534065241"} 2024-12-07T01:14:25,244 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=test, state=ENABLED in hbase:meta 2024-12-07T01:14:25,245 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_POST_OPERATION 2024-12-07T01:14:25,250 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test in 949 msec 2024-12-07T01:14:25,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46141 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-07T01:14:25,452 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:test completed 2024-12-07T01:14:25,452 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table test get assigned. Timeout = 60000ms 2024-12-07T01:14:25,455 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-07T01:14:25,478 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table test assigned to meta. Checking AM states. 2024-12-07T01:14:25,479 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-07T01:14:25,479 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table test assigned. 2024-12-07T01:14:25,480 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-12-07T01:14:25,486 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test,, stopping at row=test ,, for max=2147483647 with caching=100 2024-12-07T01:14:25,509 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test,, stopping at row=test ,, for max=2147483647 with caching=100 2024-12-07T01:14:25,522 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T01:14:25,524 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39846, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T01:14:25,526 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T01:14:25,528 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57172, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T01:14:25,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46141 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 6d8689bfa03ba7aed23b15e5ff5d23ac changed from -1.0 to 0.0, refreshing cache 2024-12-07T01:14:25,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46141 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 37d131a6ac713be02dc73a29079210f5 changed from -1.0 to 0.0, refreshing cache 2024-12-07T01:14:25,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46141 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 5d3306d206c48a1c7060e49d4fe00ad5 changed from -1.0 to 0.0, refreshing cache 2024-12-07T01:14:25,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46141 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region d2b1eb41dda894f4f7b826a2826a9c7b changed from -1.0 to 0.0, refreshing cache 2024-12-07T01:14:25,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46141 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 324bdaae20264b3880845c3082304e57 changed from -1.0 to 0.0, refreshing cache 2024-12-07T01:14:25,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46141 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region fa4e20ed49e59ee4cf63b6d84515872f changed from -1.0 to 0.0, refreshing cache 2024-12-07T01:14:25,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46141 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 80bacdcc81a44a51d622e9d873892ea3 changed from -1.0 to 0.0, refreshing cache 2024-12-07T01:14:25,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46141 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1cf1ce3ee0822ff957b8ffa13f56df7d changed from -1.0 to 0.0, refreshing cache 2024-12-07T01:14:25,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46141 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 976bbe8d7afc3b1e02af11d1b515e82c changed from -1.0 to 0.0, refreshing cache 2024-12-07T01:14:25,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46141 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region af843c9df2505d2eeee3c18062e9a2a6 changed from -1.0 to 0.0, refreshing cache 2024-12-07T01:14:25,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46141 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region c0f43810347933724c9e6319b11ae5cc changed from -1.0 to 0.0, refreshing cache 2024-12-07T01:14:25,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46141 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region aa3dca7b6c799eb77565fb0e20ef1c15 changed from -1.0 to 0.0, refreshing cache 2024-12-07T01:14:25,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46141 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 327d2bdcc20803fcfd1e61553f30068b changed from -1.0 to 0.0, refreshing cache 2024-12-07T01:14:25,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46141 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region c66f402dd9e22edaa5996a3b7446808c changed from -1.0 to 0.0, refreshing cache 2024-12-07T01:14:25,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46141 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1da36e78b320d37341113b4185c91b26 changed from -1.0 to 0.0, refreshing cache 2024-12-07T01:14:25,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46141 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 76cd5c1764feda4c992319b218572cef changed from -1.0 to 0.0, refreshing cache 2024-12-07T01:14:25,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46141 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region b59f40eb016096571b13ae445f262449 changed from -1.0 to 0.0, refreshing cache 2024-12-07T01:14:25,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46141 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 0980fd6969c12f09b843d772c626dfa5 changed from -1.0 to 0.0, refreshing cache 2024-12-07T01:14:25,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46141 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region c0a570532487ea0ebbc51cbec9ff7e37 changed from -1.0 to 0.0, refreshing cache 2024-12-07T01:14:25,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46141 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 9c426833151f173ef0d81d9c0ae11767 changed from -1.0 to 0.0, refreshing cache 2024-12-07T01:14:25,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46141 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 75c927364905f2c6bffff16328d8aaf5 changed from -1.0 to 0.0, refreshing cache 2024-12-07T01:14:25,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46141 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 3fe1b6a546d269c366ae3f13263e1a2a changed from -1.0 to 0.0, refreshing cache 2024-12-07T01:14:25,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46141 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 856730db89c49e65ccb40b5967ccf99d changed from -1.0 to 0.0, refreshing cache 2024-12-07T01:14:25,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46141 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 3f148a9f8c5bea63f4744afb36553ef7 changed from -1.0 to 0.0, refreshing cache 2024-12-07T01:14:25,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46141 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 8cd2040dc039667d68bc1a24f5f2322f changed from -1.0 to 0.0, refreshing cache 2024-12-07T01:14:25,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46141 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 73ec7ce1c93542e31d99b7b4474a9750 changed from -1.0 to 0.0, refreshing cache 2024-12-07T01:14:25,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46141 {}] rsgroup.RSGroupBasedLoadBalancer(138): Balancing RSGroup=default 2024-12-07T01:14:25,590 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46141 {}] rsgroup.RSGroupBasedLoadBalancer(151): Start Generate Balance plan for group: default 2024-12-07T01:14:25,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46141 {}] balancer.BaseLoadBalancer(619): Start Generate Balance plan for cluster. 2024-12-07T01:14:25,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46141 {}] balancer.BalancerClusterState(204): Hosts are {ec1863dc21e5=0} racks are {/default-rack=0} 2024-12-07T01:14:25,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46141 {}] balancer.BalancerClusterState(303): server 0 has 9 regions 2024-12-07T01:14:25,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46141 {}] balancer.BalancerClusterState(303): server 1 has 9 regions 2024-12-07T01:14:25,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46141 {}] balancer.BalancerClusterState(303): server 2 has 9 regions 2024-12-07T01:14:25,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46141 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-07T01:14:25,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46141 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-07T01:14:25,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46141 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-12-07T01:14:25,591 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46141 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-07T01:14:25,592 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46141 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-07T01:14:25,592 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46141 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-12-07T01:14:25,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46141 {}] balancer.BalancerClusterState(326): Number of tables=2, number of hosts=1, number of racks=1 2024-12-07T01:14:25,596 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46141 {}] balancer.StochasticLoadBalancer(395): Cluster wide - skipping load balancing because weighted average imbalance=0.016474464579901153 <= threshold(0.025). If you want more aggressive balancing, either lower hbase.master.balancer.stochastic.minCostNeedBalance from 0.025 or increase the relative multiplier(s) of the specific cost function(s). functionCost=RegionCountSkewCostFunction : (multiplier=500.0, imbalance=0.0); PrimaryRegionCountSkewCostFunction : (not needed); MoveCostFunction : (multiplier=7.0, imbalance=0.0); ServerLocalityCostFunction : (multiplier=25.0, imbalance=0.0); RackLocalityCostFunction : (multiplier=15.0, imbalance=0.0); TableSkewCostFunction : (multiplier=35.0, imbalance=0.0); RegionReplicaHostCostFunction : (not needed); RegionReplicaRackCostFunction : (not needed); ReadRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); CPRequestCostFunction : (multiplier=5.0, imbalance=0.0); WriteRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); MemStoreSizeCostFunction : (multiplier=5.0, imbalance=0.0); StoreFileCostFunction : (multiplier=5.0, imbalance=0.0); 2024-12-07T01:14:25,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46141 {}] master.HMaster(2203): Balancer is going into sleep until next period in 300000ms 2024-12-07T01:14:25,598 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [900,000] milli-secs(wait.for.ratio=[1]) 2024-12-07T01:14:25,606 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46141 {}] master.MasterRpcServices(567): Client=jenkins//172.17.0.3 set balanceSwitch=false 2024-12-07T01:14:25,624 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: master.assignment.TestRollbackSCP#testFailAndRollback Thread=382, OpenFileDescriptor=649, MaxFileDescriptor=1048576, SystemLoadAverage=443, ProcessCount=11, AvailableMemoryMB=242 2024-12-07T01:14:25,627 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table hbase:meta 2024-12-07T01:14:25,627 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=hbase:meta,,1 2024-12-07T01:14:25,628 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(267): Killing ec1863dc21e5,36079,1733534061701 2024-12-07T01:14:25,628 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server ec1863dc21e5,36079,1733534061701: Simulated kill ***** 2024-12-07T01:14:25,628 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-12-07T01:14:25,631 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-12-07T01:14:25,633 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-12-07T01:14:25,633 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-12-07T01:14:25,633 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-12-07T01:14:25,634 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 297018816 }, "NonHeapMemoryUsage": { "committed": 150994944, "init": 7667712, "max": -1, "used": 148509536 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "ec1863dc21e5" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "ec1863dc21e5" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.Context": "regionserver", "tag.Hostname": "ec1863dc21e5" } ] } 2024-12-07T01:14:25,637 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46141 {}] master.MasterRpcServices(700): ec1863dc21e5,36079,1733534061701 reported a fatal error: ***** ABORTING region server ec1863dc21e5,36079,1733534061701: Simulated kill ***** 2024-12-07T01:14:25,638 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'ec1863dc21e5,36079,1733534061701' ***** 2024-12-07T01:14:25,638 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Simulated kill 2024-12-07T01:14:25,639 INFO [RS:1;ec1863dc21e5:36079 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-07T01:14:25,639 INFO [RS:1;ec1863dc21e5:36079 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-12-07T01:14:25,639 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-07T01:14:25,639 INFO [RS:1;ec1863dc21e5:36079 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-12-07T01:14:25,639 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [15,000] milli-secs(wait.for.ratio=[1]) 2024-12-07T01:14:25,639 INFO [RS:1;ec1863dc21e5:36079 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-07T01:14:25,639 INFO [RS:1;ec1863dc21e5:36079 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;ec1863dc21e5:36079. 2024-12-07T01:14:25,639 DEBUG [RS:1;ec1863dc21e5:36079 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-07T01:14:25,644 DEBUG [RS:1;ec1863dc21e5:36079 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T01:14:25,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43991 is added to blk_1073741836_1012 (size=32114) 2024-12-07T01:14:25,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741836_1012 (size=32114) 2024-12-07T01:14:25,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37235 is added to blk_1073741836_1012 (size=32114) 2024-12-07T01:14:25,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741834_1010 (size=2237) 2024-12-07T01:14:25,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43991 is added to blk_1073741834_1010 (size=2237) 2024-12-07T01:14:25,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37235 is added to blk_1073741834_1010 (size=2237) 2024-12-07T01:14:25,653 DEBUG [RS:1;ec1863dc21e5:36079 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T01:14:25,654 INFO [RS:1;ec1863dc21e5:36079 {}] regionserver.LeaseManager(133): Closed leases 2024-12-07T01:14:25,654 INFO [RS:1;ec1863dc21e5:36079 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:36079 2024-12-07T01:14:25,694 INFO [regionserver/ec1863dc21e5:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-07T01:14:25,728 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46141-0x101ad5a899b0000, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-07T01:14:25,728 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36079-0x101ad5a899b0002, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/ec1863dc21e5,36079,1733534061701 2024-12-07T01:14:25,728 INFO [RS:1;ec1863dc21e5:36079 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-07T01:14:25,729 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [ec1863dc21e5,36079,1733534061701] 2024-12-07T01:14:25,748 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/ec1863dc21e5,36079,1733534061701 already deleted, retry=false 2024-12-07T01:14:25,749 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of ec1863dc21e5,36079,1733534061701 on ec1863dc21e5,46141,1733534060840 2024-12-07T01:14:25,756 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=57, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure ec1863dc21e5,36079,1733534061701, splitWal=true, meta=true 2024-12-07T01:14:25,760 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=57 for ec1863dc21e5,36079,1733534061701 (carryingMeta=true) ec1863dc21e5,36079,1733534061701/CRASHED/regionCount=9/lock=java.util.concurrent.locks.ReentrantReadWriteLock@76c9bec4[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-12-07T01:14:25,761 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=57, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure ec1863dc21e5,36079,1733534061701, splitWal=true, meta=true 2024-12-07T01:14:25,762 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure ec1863dc21e5,36079,1733534061701, splitWal=true, meta=true, isMeta: true 2024-12-07T01:14:25,765 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/WALs/ec1863dc21e5,36079,1733534061701-splitting 2024-12-07T01:14:25,766 INFO [PEWorker-1 {}] master.SplitWALManager(105): ec1863dc21e5,36079,1733534061701 WAL count=1, meta=true 2024-12-07T01:14:25,770 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure ec1863dc21e5%2C36079%2C1733534061701.meta.1733534063927.meta}] 2024-12-07T01:14:25,779 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:25,783 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure ec1863dc21e5%2C36079%2C1733534061701.meta.1733534063927.meta, worker=ec1863dc21e5,42915,1733534061580}] 2024-12-07T01:14:25,841 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-12-07T01:14:25,841 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36079-0x101ad5a899b0002, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-07T01:14:25,841 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36079-0x101ad5a899b0002, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-07T01:14:25,842 INFO [RS:1;ec1863dc21e5:36079 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-07T01:14:25,842 INFO [RS:1;ec1863dc21e5:36079 {}] regionserver.HRegionServer(1031): Exiting; stopping=ec1863dc21e5,36079,1733534061701; zookeeper connection closed. 2024-12-07T01:14:25,842 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@50f6ac46 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@50f6ac46 2024-12-07T01:14:25,950 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42915 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=59 2024-12-07T01:14:25,971 INFO [RS_LOG_REPLAY_OPS-regionserver/ec1863dc21e5:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(299): Splitting hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/WALs/ec1863dc21e5,36079,1733534061701-splitting/ec1863dc21e5%2C36079%2C1733534061701.meta.1733534063927.meta, size=31.4 K (32114bytes) 2024-12-07T01:14:25,972 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ec1863dc21e5:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-12-07T01:14:25,972 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ec1863dc21e5:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-12-07T01:14:25,972 INFO [RS_LOG_REPLAY_OPS-regionserver/ec1863dc21e5:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/WALs/ec1863dc21e5,36079,1733534061701-splitting/ec1863dc21e5%2C36079%2C1733534061701.meta.1733534063927.meta 2024-12-07T01:14:25,977 INFO [RS_LOG_REPLAY_OPS-regionserver/ec1863dc21e5:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/WALs/ec1863dc21e5,36079,1733534061701-splitting/ec1863dc21e5%2C36079%2C1733534061701.meta.1733534063927.meta after 4ms 2024-12-07T01:14:25,985 INFO [RS_LOG_REPLAY_OPS-regionserver/ec1863dc21e5:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(310): Open hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/WALs/ec1863dc21e5,36079,1733534061701-splitting/ec1863dc21e5%2C36079%2C1733534061701.meta.1733534063927.meta took 15ms 2024-12-07T01:14:25,994 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ec1863dc21e5:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(352): Last flushed sequenceid for 1588230740: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "info" sequence_id: 5 } store_sequence_id { family_name: "ns" sequence_id: 3 } store_sequence_id { family_name: "rep_barrier" sequence_id: 18446744073709551615 } store_sequence_id { family_name: "table" sequence_id: 6 } 2024-12-07T01:14:25,999 INFO [RS_LOG_REPLAY_OPS-regionserver/ec1863dc21e5:0-0-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-ec1863dc21e5%2C36079%2C1733534061701.meta.1733534063927.meta.temp 2024-12-07T01:14:26,000 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ec1863dc21e5:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/WALs/ec1863dc21e5,36079,1733534061701-splitting/ec1863dc21e5%2C36079%2C1733534061701.meta.1733534063927.meta so closing down 2024-12-07T01:14:26,001 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ec1863dc21e5:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-07T01:14:26,003 INFO [RS_LOG_REPLAY_OPS-regionserver/ec1863dc21e5:0-0-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-ec1863dc21e5%2C36079%2C1733534061701.meta.1733534063927.meta.temp 2024-12-07T01:14:26,007 INFO [RS_LOG_REPLAY_OPS-regionserver/ec1863dc21e5:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.OutputSink(145): 3 split writer threads finished 2024-12-07T01:14:26,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741864_1040 (size=31873) 2024-12-07T01:14:26,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37235 is added to blk_1073741864_1040 (size=31873) 2024-12-07T01:14:26,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43991 is added to blk_1073741864_1040 (size=31873) 2024-12-07T01:14:26,019 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-ec1863dc21e5%2C36079%2C1733534061701.meta.1733534063927.meta.temp (wrote 57 edits, skipped 0 edits in 2 ms) 2024-12-07T01:14:26,022 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-ec1863dc21e5%2C36079%2C1733534061701.meta.1733534063927.meta.temp to hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-12-07T01:14:26,025 INFO [RS_LOG_REPLAY_OPS-regionserver/ec1863dc21e5:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(425): Processed 58 edits across 1 Regions in 36 ms; skipped=1; WAL=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/WALs/ec1863dc21e5,36079,1733534061701-splitting/ec1863dc21e5%2C36079%2C1733534061701.meta.1733534063927.meta, size=31.4 K, length=32114, corrupted=false, cancelled=false 2024-12-07T01:14:26,026 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ec1863dc21e5:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(428): Completed split of hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/WALs/ec1863dc21e5,36079,1733534061701-splitting/ec1863dc21e5%2C36079%2C1733534061701.meta.1733534063927.meta, journal: Splitting hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/WALs/ec1863dc21e5,36079,1733534061701-splitting/ec1863dc21e5%2C36079%2C1733534061701.meta.1733534063927.meta, size=31.4 K (32114bytes) at 1733534065971Finishing writing output for hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/WALs/ec1863dc21e5,36079,1733534061701-splitting/ec1863dc21e5%2C36079%2C1733534061701.meta.1733534063927.meta so closing down at 1733534066001 (+30 ms)Creating recovered edits writer path=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-ec1863dc21e5%2C36079%2C1733534061701.meta.1733534063927.meta.temp at 1733534066003 (+2 ms)3 split writer threads finished at 1733534066007 (+4 ms)Closed recovered edits writer path=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-ec1863dc21e5%2C36079%2C1733534061701.meta.1733534063927.meta.temp (wrote 57 edits, skipped 0 edits in 2 ms) at 1733534066019 (+12 ms)Rename recovered edits hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-ec1863dc21e5%2C36079%2C1733534061701.meta.1733534063927.meta.temp to hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 at 1733534066022 (+3 ms)Processed 58 edits across 1 Regions in 36 ms; skipped=1; WAL=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/WALs/ec1863dc21e5,36079,1733534061701-splitting/ec1863dc21e5%2C36079%2C1733534061701.meta.1733534063927.meta, size=31.4 K, length=32114, corrupted=false, cancelled=false at 1733534066025 (+3 ms) 2024-12-07T01:14:26,026 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ec1863dc21e5:0-0 {event_type=RS_LOG_REPLAY, pid=59}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/WALs/ec1863dc21e5,36079,1733534061701-splitting/ec1863dc21e5%2C36079%2C1733534061701.meta.1733534063927.meta 2024-12-07T01:14:26,027 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ec1863dc21e5:0-0 {event_type=RS_LOG_REPLAY, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-12-07T01:14:26,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46141 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-12-07T01:14:26,035 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/WALs/ec1863dc21e5,36079,1733534061701-splitting/ec1863dc21e5%2C36079%2C1733534061701.meta.1733534063927.meta to hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/oldWALs 2024-12-07T01:14:26,038 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-12-07T01:14:26,038 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ec1863dc21e5%2C36079%2C1733534061701.meta.1733534063927.meta, worker=ec1863dc21e5,42915,1733534061580 in 253 msec 2024-12-07T01:14:26,040 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:26,043 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-12-07T01:14:26,043 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ec1863dc21e5%2C36079%2C1733534061701.meta.1733534063927.meta, worker=ec1863dc21e5,42915,1733534061580 in 272 msec 2024-12-07T01:14:26,045 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/WALs/ec1863dc21e5,36079,1733534061701-splitting dir is empty, no logs to split. 2024-12-07T01:14:26,045 INFO [PEWorker-1 {}] master.SplitWALManager(105): ec1863dc21e5,36079,1733534061701 WAL count=0, meta=true 2024-12-07T01:14:26,045 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if ec1863dc21e5,36079,1733534061701 WAL splitting is done? wals=0, meta=true 2024-12-07T01:14:26,047 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-07T01:14:26,049 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-07T01:14:26,050 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-07T01:14:26,200 DEBUG [ec1863dc21e5:46141 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-12-07T01:14:26,201 DEBUG [ec1863dc21e5:46141 {}] balancer.BalancerClusterState(204): Hosts are {ec1863dc21e5=0} racks are {/default-rack=0} 2024-12-07T01:14:26,201 DEBUG [ec1863dc21e5:46141 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-07T01:14:26,201 DEBUG [ec1863dc21e5:46141 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-07T01:14:26,201 DEBUG [ec1863dc21e5:46141 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-07T01:14:26,201 DEBUG [ec1863dc21e5:46141 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-07T01:14:26,201 INFO [ec1863dc21e5:46141 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-07T01:14:26,201 INFO [ec1863dc21e5:46141 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-07T01:14:26,201 DEBUG [ec1863dc21e5:46141 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-07T01:14:26,202 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=ec1863dc21e5,40693,1733534061744 2024-12-07T01:14:26,205 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ec1863dc21e5,40693,1733534061744, state=OPENING 2024-12-07T01:14:26,254 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42915-0x101ad5a899b0001, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T01:14:26,254 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40693-0x101ad5a899b0003, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T01:14:26,254 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46141-0x101ad5a899b0000, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T01:14:26,254 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-07T01:14:26,254 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T01:14:26,254 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T01:14:26,254 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=ec1863dc21e5,40693,1733534061744}] 2024-12-07T01:14:26,254 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T01:14:26,323 INFO [regionserver/ec1863dc21e5:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-07T01:14:26,323 INFO [regionserver/ec1863dc21e5:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-07T01:14:26,414 INFO [RS_OPEN_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_META, pid=61}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-07T01:14:26,414 INFO [RS_OPEN_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-07T01:14:26,414 INFO [RS_OPEN_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_META, pid=61}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-07T01:14:26,416 INFO [RS_OPEN_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ec1863dc21e5%2C40693%2C1733534061744.meta, suffix=.meta, logDir=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/WALs/ec1863dc21e5,40693,1733534061744, archiveDir=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/oldWALs, maxLogs=32 2024-12-07T01:14:26,429 DEBUG [RS_OPEN_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_META, pid=61}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/WALs/ec1863dc21e5,40693,1733534061744/ec1863dc21e5%2C40693%2C1733534061744.meta.1733534066417.meta, exclude list is [], retry=0 2024-12-07T01:14:26,434 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43991,DS-f5ad7f4d-d62b-424d-bad1-dbf9e5694bea,DISK] 2024-12-07T01:14:26,434 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40127,DS-59048819-6295-483c-8edd-efcadeb66bf1,DISK] 2024-12-07T01:14:26,436 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37235,DS-b9cb305a-a2e1-439c-816b-c45d44c27920,DISK] 2024-12-07T01:14:26,441 INFO [RS_OPEN_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/WALs/ec1863dc21e5,40693,1733534061744/ec1863dc21e5%2C40693%2C1733534061744.meta.1733534066417.meta 2024-12-07T01:14:26,441 DEBUG [RS_OPEN_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45341:45341),(127.0.0.1/127.0.0.1:40865:40865),(127.0.0.1/127.0.0.1:39885:39885)] 2024-12-07T01:14:26,442 DEBUG [RS_OPEN_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-07T01:14:26,442 DEBUG [RS_OPEN_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_META, pid=61}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-07T01:14:26,442 DEBUG [RS_OPEN_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-07T01:14:26,442 INFO [RS_OPEN_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-07T01:14:26,442 DEBUG [RS_OPEN_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-07T01:14:26,442 DEBUG [RS_OPEN_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T01:14:26,442 DEBUG [RS_OPEN_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-07T01:14:26,442 DEBUG [RS_OPEN_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-07T01:14:26,444 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-07T01:14:26,445 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-07T01:14:26,445 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:14:26,446 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T01:14:26,446 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-07T01:14:26,447 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-07T01:14:26,447 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:14:26,447 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T01:14:26,447 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-07T01:14:26,448 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-07T01:14:26,448 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:14:26,449 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T01:14:26,449 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-07T01:14:26,450 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-07T01:14:26,450 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:14:26,450 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T01:14:26,450 DEBUG [RS_OPEN_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-07T01:14:26,451 DEBUG [RS_OPEN_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/hbase/meta/1588230740 2024-12-07T01:14:26,454 DEBUG [RS_OPEN_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/hbase/meta/1588230740 2024-12-07T01:14:26,456 INFO [RS_OPEN_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-12-07T01:14:26,469 DEBUG [RS_OPEN_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5793): Applied 264, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=60, path=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-12-07T01:14:26,472 INFO [RS_OPEN_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=28 KB heapSize=48.31 KB 2024-12-07T01:14:26,526 DEBUG [RS_OPEN_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_META, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/hbase/meta/1588230740/.tmp/info/f9799b47579847c9b0e93168d1b6e73e is 138, key is test,bbb,1733534064285.0980fd6969c12f09b843d772c626dfa5./info:regioninfo/1733534064948/Put/seqid=0 2024-12-07T01:14:26,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37235 is added to blk_1073741866_1042 (size=34924) 2024-12-07T01:14:26,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741866_1042 (size=34924) 2024-12-07T01:14:26,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43991 is added to blk_1073741866_1042 (size=34924) 2024-12-07T01:14:26,537 INFO [RS_OPEN_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=27.86 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/hbase/meta/1588230740/.tmp/info/f9799b47579847c9b0e93168d1b6e73e 2024-12-07T01:14:26,591 DEBUG [RS_OPEN_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_META, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/hbase/meta/1588230740/.tmp/ns/e62a68286a584d6f8ad981142e61f983 is 43, key is default/ns:d/1733534064071/Put/seqid=0 2024-12-07T01:14:26,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741867_1043 (size=5153) 2024-12-07T01:14:26,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43991 is added to blk_1073741867_1043 (size=5153) 2024-12-07T01:14:26,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37235 is added to blk_1073741867_1043 (size=5153) 2024-12-07T01:14:26,599 INFO [RS_OPEN_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/hbase/meta/1588230740/.tmp/ns/e62a68286a584d6f8ad981142e61f983 2024-12-07T01:14:26,618 DEBUG [RS_OPEN_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_META, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/hbase/meta/1588230740/.tmp/table/88708d8c00164acab13461bccac56816 is 40, key is test/table:state/1733534065241/Put/seqid=0 2024-12-07T01:14:26,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741868_1044 (size=5165) 2024-12-07T01:14:26,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37235 is added to blk_1073741868_1044 (size=5165) 2024-12-07T01:14:26,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43991 is added to blk_1073741868_1044 (size=5165) 2024-12-07T01:14:26,625 INFO [RS_OPEN_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=72 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/hbase/meta/1588230740/.tmp/table/88708d8c00164acab13461bccac56816 2024-12-07T01:14:26,634 DEBUG [RS_OPEN_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/hbase/meta/1588230740/.tmp/info/f9799b47579847c9b0e93168d1b6e73e as hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/hbase/meta/1588230740/info/f9799b47579847c9b0e93168d1b6e73e 2024-12-07T01:14:26,643 INFO [RS_OPEN_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/hbase/meta/1588230740/info/f9799b47579847c9b0e93168d1b6e73e, entries=260, sequenceid=60, filesize=34.1 K 2024-12-07T01:14:26,646 DEBUG [RS_OPEN_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/hbase/meta/1588230740/.tmp/ns/e62a68286a584d6f8ad981142e61f983 as hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/hbase/meta/1588230740/ns/e62a68286a584d6f8ad981142e61f983 2024-12-07T01:14:26,654 INFO [RS_OPEN_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/hbase/meta/1588230740/ns/e62a68286a584d6f8ad981142e61f983, entries=2, sequenceid=60, filesize=5.0 K 2024-12-07T01:14:26,655 DEBUG [RS_OPEN_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/hbase/meta/1588230740/.tmp/table/88708d8c00164acab13461bccac56816 as hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/hbase/meta/1588230740/table/88708d8c00164acab13461bccac56816 2024-12-07T01:14:26,663 INFO [RS_OPEN_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/hbase/meta/1588230740/table/88708d8c00164acab13461bccac56816, entries=2, sequenceid=60, filesize=5.0 K 2024-12-07T01:14:26,666 INFO [RS_OPEN_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~28 KB/28672, heapSize ~48.02 KB/49168, currentSize=0 B/0 for 1588230740 in 193ms, sequenceid=60, compaction requested=false; wal=null 2024-12-07T01:14:26,667 DEBUG [RS_OPEN_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-07T01:14:26,670 DEBUG [RS_OPEN_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-12-07T01:14:26,671 DEBUG [RS_OPEN_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-07T01:14:26,671 DEBUG [RS_OPEN_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-07T01:14:26,672 DEBUG [RS_OPEN_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-07T01:14:26,674 DEBUG [RS_OPEN_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-07T01:14:26,676 DEBUG [RS_OPEN_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/hbase/meta/1588230740/recovered.edits/60.seqid, newMaxSeqId=60, maxSeqId=1 2024-12-07T01:14:26,678 INFO [RS_OPEN_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=61; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62860893, jitterRate=-0.06329970061779022}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-07T01:14:26,678 DEBUG [RS_OPEN_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-07T01:14:26,678 DEBUG [RS_OPEN_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733534066442Writing region info on filesystem at 1733534066442Initializing all the Stores at 1733534066444 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733534066444Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733534066444Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534066444Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733534066444Obtaining lock to block concurrent updates at 1733534066472 (+28 ms)Preparing flush snapshotting stores in 1588230740 at 1733534066472Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=28672, getHeapSize=49408, getOffHeapSize=0, getCellsCount=264 at 1733534066475 (+3 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733534066475Flushing 1588230740/info: creating writer at 1733534066476 (+1 ms)Flushing 1588230740/info: appending metadata at 1733534066518 (+42 ms)Flushing 1588230740/info: closing flushed file at 1733534066520 (+2 ms)Flushing 1588230740/ns: creating writer at 1733534066577 (+57 ms)Flushing 1588230740/ns: appending metadata at 1733534066591 (+14 ms)Flushing 1588230740/ns: closing flushed file at 1733534066591Flushing 1588230740/table: creating writer at 1733534066605 (+14 ms)Flushing 1588230740/table: appending metadata at 1733534066618 (+13 ms)Flushing 1588230740/table: closing flushed file at 1733534066618Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@70572dba: reopening flushed file at 1733534066633 (+15 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@116bad2b: reopening flushed file at 1733534066645 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@33517829: reopening flushed file at 1733534066654 (+9 ms)Finished flush of dataSize ~28 KB/28672, heapSize ~48.02 KB/49168, currentSize=0 B/0 for 1588230740 in 193ms, sequenceid=60, compaction requested=false; wal=null at 1733534066666 (+12 ms)Cleaning up temporary data from old regions at 1733534066671 (+5 ms)Running coprocessor post-open hooks at 1733534066678 (+7 ms)Region opened successfully at 1733534066678 2024-12-07T01:14:26,679 INFO [RS_OPEN_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=61, masterSystemTime=1733534066407 2024-12-07T01:14:26,681 DEBUG [RS_OPEN_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-07T01:14:26,681 INFO [RS_OPEN_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_META, pid=61}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-07T01:14:26,683 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=61, regionLocation=ec1863dc21e5,40693,1733534061744 2024-12-07T01:14:26,684 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ec1863dc21e5,40693,1733534061744, state=OPEN 2024-12-07T01:14:26,733 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46141 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=ec1863dc21e5,40693,1733534061744, table=hbase:meta, region=1588230740. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-07T01:14:26,769 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42915-0x101ad5a899b0001, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T01:14:26,769 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40693-0x101ad5a899b0003, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T01:14:26,769 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46141-0x101ad5a899b0000, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T01:14:26,770 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=61, ppid=60, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=ec1863dc21e5,40693,1733534061744 2024-12-07T01:14:26,770 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T01:14:26,770 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T01:14:26,770 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T01:14:26,773 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-12-07T01:14:26,773 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ec1863dc21e5,40693,1733534061744 in 516 msec 2024-12-07T01:14:26,775 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=60, resume processing ppid=57 2024-12-07T01:14:26,775 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 726 msec 2024-12-07T01:14:26,776 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): ec1863dc21e5,36079,1733534061701 had 9 regions 2024-12-07T01:14:26,777 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure ec1863dc21e5,36079,1733534061701, splitWal=true, meta=true, isMeta: false 2024-12-07T01:14:26,779 INFO [PEWorker-1 {}] master.SplitWALManager(105): ec1863dc21e5,36079,1733534061701 WAL count=1, meta=false 2024-12-07T01:14:26,780 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=57, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure ec1863dc21e5%2C36079%2C1733534061701.1733534063448}] 2024-12-07T01:14:26,782 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:26,783 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure ec1863dc21e5%2C36079%2C1733534061701.1733534063448, worker=ec1863dc21e5,42915,1733534061580}] 2024-12-07T01:14:26,937 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42915 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=63 2024-12-07T01:14:26,957 INFO [RS_LOG_REPLAY_OPS-regionserver/ec1863dc21e5:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(299): Splitting hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/WALs/ec1863dc21e5,36079,1733534061701-splitting/ec1863dc21e5%2C36079%2C1733534061701.1733534063448, size=2.2 K (2237bytes) 2024-12-07T01:14:26,957 INFO [RS_LOG_REPLAY_OPS-regionserver/ec1863dc21e5:0-1 {event_type=RS_LOG_REPLAY, pid=63}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/WALs/ec1863dc21e5,36079,1733534061701-splitting/ec1863dc21e5%2C36079%2C1733534061701.1733534063448 2024-12-07T01:14:26,958 INFO [RS_LOG_REPLAY_OPS-regionserver/ec1863dc21e5:0-1 {event_type=RS_LOG_REPLAY, pid=63}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/WALs/ec1863dc21e5,36079,1733534061701-splitting/ec1863dc21e5%2C36079%2C1733534061701.1733534063448 after 1ms 2024-12-07T01:14:26,961 INFO [RS_LOG_REPLAY_OPS-regionserver/ec1863dc21e5:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(310): Open hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/WALs/ec1863dc21e5,36079,1733534061701-splitting/ec1863dc21e5%2C36079%2C1733534061701.1733534063448 took 4ms 2024-12-07T01:14:26,964 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ec1863dc21e5:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 80bacdcc81a44a51d622e9d873892ea3: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-12-07T01:14:26,966 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ec1863dc21e5:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 5d3306d206c48a1c7060e49d4fe00ad5: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-12-07T01:14:26,968 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ec1863dc21e5:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 6d8689bfa03ba7aed23b15e5ff5d23ac: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-12-07T01:14:26,969 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ec1863dc21e5:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 1cf1ce3ee0822ff957b8ffa13f56df7d: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-12-07T01:14:26,971 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ec1863dc21e5:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for d2b1eb41dda894f4f7b826a2826a9c7b: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-12-07T01:14:26,972 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ec1863dc21e5:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 324bdaae20264b3880845c3082304e57: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-12-07T01:14:26,974 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ec1863dc21e5:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for fa4e20ed49e59ee4cf63b6d84515872f: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-12-07T01:14:26,975 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ec1863dc21e5:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 37d131a6ac713be02dc73a29079210f5: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-12-07T01:14:26,975 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ec1863dc21e5:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/WALs/ec1863dc21e5,36079,1733534061701-splitting/ec1863dc21e5%2C36079%2C1733534061701.1733534063448 so closing down 2024-12-07T01:14:26,975 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ec1863dc21e5:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-07T01:14:26,975 INFO [RS_LOG_REPLAY_OPS-regionserver/ec1863dc21e5:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.OutputSink(145): 3 split writer threads finished 2024-12-07T01:14:26,976 INFO [RS_LOG_REPLAY_OPS-regionserver/ec1863dc21e5:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(425): Processed 8 edits across 0 Regions in 14 ms; skipped=8; WAL=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/WALs/ec1863dc21e5,36079,1733534061701-splitting/ec1863dc21e5%2C36079%2C1733534061701.1733534063448, size=2.2 K, length=2237, corrupted=false, cancelled=false 2024-12-07T01:14:26,976 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ec1863dc21e5:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(428): Completed split of hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/WALs/ec1863dc21e5,36079,1733534061701-splitting/ec1863dc21e5%2C36079%2C1733534061701.1733534063448, journal: Splitting hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/WALs/ec1863dc21e5,36079,1733534061701-splitting/ec1863dc21e5%2C36079%2C1733534061701.1733534063448, size=2.2 K (2237bytes) at 1733534066957Finishing writing output for hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/WALs/ec1863dc21e5,36079,1733534061701-splitting/ec1863dc21e5%2C36079%2C1733534061701.1733534063448 so closing down at 1733534066975 (+18 ms)3 split writer threads finished at 1733534066976 (+1 ms)Processed 8 edits across 0 Regions in 14 ms; skipped=8; WAL=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/WALs/ec1863dc21e5,36079,1733534061701-splitting/ec1863dc21e5%2C36079%2C1733534061701.1733534063448, size=2.2 K, length=2237, corrupted=false, cancelled=false at 1733534066976 2024-12-07T01:14:26,976 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ec1863dc21e5:0-1 {event_type=RS_LOG_REPLAY, pid=63}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/WALs/ec1863dc21e5,36079,1733534061701-splitting/ec1863dc21e5%2C36079%2C1733534061701.1733534063448 2024-12-07T01:14:26,976 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ec1863dc21e5:0-1 {event_type=RS_LOG_REPLAY, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-12-07T01:14:26,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46141 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-12-07T01:14:26,981 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/WALs/ec1863dc21e5,36079,1733534061701-splitting/ec1863dc21e5%2C36079%2C1733534061701.1733534063448 to hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/oldWALs 2024-12-07T01:14:26,983 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-12-07T01:14:26,983 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ec1863dc21e5%2C36079%2C1733534061701.1733534063448, worker=ec1863dc21e5,42915,1733534061580 in 198 msec 2024-12-07T01:14:26,984 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:26,986 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=57 2024-12-07T01:14:26,987 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ec1863dc21e5%2C36079%2C1733534061701.1733534063448, worker=ec1863dc21e5,42915,1733534061580 in 204 msec 2024-12-07T01:14:26,988 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/WALs/ec1863dc21e5,36079,1733534061701-splitting dir is empty, no logs to split. 2024-12-07T01:14:26,988 INFO [PEWorker-1 {}] master.SplitWALManager(105): ec1863dc21e5,36079,1733534061701 WAL count=0, meta=false 2024-12-07T01:14:26,988 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if ec1863dc21e5,36079,1733534061701 WAL splitting is done? wals=0, meta=false 2024-12-07T01:14:26,991 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=37d131a6ac713be02dc73a29079210f5, ASSIGN}, {pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=324bdaae20264b3880845c3082304e57, ASSIGN}, {pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=1cf1ce3ee0822ff957b8ffa13f56df7d, ASSIGN}, {pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=6d8689bfa03ba7aed23b15e5ff5d23ac, ASSIGN}, {pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=5d3306d206c48a1c7060e49d4fe00ad5, ASSIGN}, {pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=80bacdcc81a44a51d622e9d873892ea3, ASSIGN}, {pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d2b1eb41dda894f4f7b826a2826a9c7b, ASSIGN}, {pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=fa4e20ed49e59ee4cf63b6d84515872f, ASSIGN}] 2024-12-07T01:14:26,992 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=fa4e20ed49e59ee4cf63b6d84515872f, ASSIGN 2024-12-07T01:14:26,993 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=fa4e20ed49e59ee4cf63b6d84515872f, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-07T01:14:26,994 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d2b1eb41dda894f4f7b826a2826a9c7b, ASSIGN 2024-12-07T01:14:26,995 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=d2b1eb41dda894f4f7b826a2826a9c7b, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-07T01:14:26,995 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=80bacdcc81a44a51d622e9d873892ea3, ASSIGN 2024-12-07T01:14:26,996 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=80bacdcc81a44a51d622e9d873892ea3, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-07T01:14:26,997 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=5d3306d206c48a1c7060e49d4fe00ad5, ASSIGN 2024-12-07T01:14:26,998 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=5d3306d206c48a1c7060e49d4fe00ad5, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-07T01:14:26,998 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=6d8689bfa03ba7aed23b15e5ff5d23ac, ASSIGN 2024-12-07T01:14:26,999 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=6d8689bfa03ba7aed23b15e5ff5d23ac, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-07T01:14:27,000 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=1cf1ce3ee0822ff957b8ffa13f56df7d, ASSIGN 2024-12-07T01:14:27,001 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=1cf1ce3ee0822ff957b8ffa13f56df7d, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-07T01:14:27,002 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=324bdaae20264b3880845c3082304e57, ASSIGN 2024-12-07T01:14:27,002 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=324bdaae20264b3880845c3082304e57, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-07T01:14:27,003 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=37d131a6ac713be02dc73a29079210f5, ASSIGN 2024-12-07T01:14:27,004 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=37d131a6ac713be02dc73a29079210f5, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-07T01:14:27,144 DEBUG [ec1863dc21e5:46141 {}] balancer.BalancerClusterState(204): Hosts are {ec1863dc21e5=0} racks are {/default-rack=0} 2024-12-07T01:14:27,144 DEBUG [ec1863dc21e5:46141 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-07T01:14:27,144 DEBUG [ec1863dc21e5:46141 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-07T01:14:27,144 DEBUG [ec1863dc21e5:46141 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-07T01:14:27,144 DEBUG [ec1863dc21e5:46141 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-07T01:14:27,145 INFO [ec1863dc21e5:46141 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-07T01:14:27,145 INFO [ec1863dc21e5:46141 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-07T01:14:27,145 DEBUG [ec1863dc21e5:46141 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-07T01:14:27,145 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=69 updating hbase:meta row=80bacdcc81a44a51d622e9d873892ea3, regionState=OPENING, regionLocation=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:27,146 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=65 updating hbase:meta row=324bdaae20264b3880845c3082304e57, regionState=OPENING, regionLocation=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:27,147 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=68 updating hbase:meta row=5d3306d206c48a1c7060e49d4fe00ad5, regionState=OPENING, regionLocation=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:27,148 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=67 updating hbase:meta row=6d8689bfa03ba7aed23b15e5ff5d23ac, regionState=OPENING, regionLocation=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:27,147 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server ec1863dc21e5:36079 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ec1863dc21e5/172.17.0.3:36079 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T01:14:27,149 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=66 updating hbase:meta row=1cf1ce3ee0822ff957b8ffa13f56df7d, regionState=OPENING, regionLocation=ec1863dc21e5,40693,1733534061744 2024-12-07T01:14:27,149 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=71 updating hbase:meta row=fa4e20ed49e59ee4cf63b6d84515872f, regionState=OPENING, regionLocation=ec1863dc21e5,40693,1733534061744 2024-12-07T01:14:27,150 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=70 updating hbase:meta row=d2b1eb41dda894f4f7b826a2826a9c7b, regionState=OPENING, regionLocation=ec1863dc21e5,40693,1733534061744 2024-12-07T01:14:27,151 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=ec1863dc21e5,36079,1733534061701, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=ec1863dc21e5,36079,1733534061701, seqNum=-1, error=java.net.ConnectException: Call to address=ec1863dc21e5:36079 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ec1863dc21e5/172.17.0.3:36079 2024-12-07T01:14:27,151 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=37d131a6ac713be02dc73a29079210f5, regionState=OPENING, regionLocation=ec1863dc21e5,40693,1733534061744 2024-12-07T01:14:27,152 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=ec1863dc21e5,36079,1733534061701, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-12-07T01:14:27,152 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=ec1863dc21e5,36079,1733534061701, seqNum=-1 from cache 2024-12-07T01:14:27,153 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address ec1863dc21e5:36079 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ec1863dc21e5/172.17.0.3:36079 2024-12-07T01:14:27,154 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server ec1863dc21e5:36079 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ec1863dc21e5/172.17.0.3:36079 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T01:14:27,154 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=ec1863dc21e5,36079,1733534061701, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=ec1863dc21e5:36079 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ec1863dc21e5/172.17.0.3:36079 2024-12-07T01:14:27,154 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=ec1863dc21e5,36079,1733534061701, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=ec1863dc21e5:36079 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ec1863dc21e5/172.17.0.3:36079 2024-12-07T01:14:27,154 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=ec1863dc21e5,36079,1733534061701, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=ec1863dc21e5:36079 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ec1863dc21e5/172.17.0.3:36079 2024-12-07T01:14:27,155 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=ec1863dc21e5,36079,1733534061701, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=ec1863dc21e5:36079 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ec1863dc21e5/172.17.0.3:36079 2024-12-07T01:14:27,155 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=ec1863dc21e5,36079,1733534061701, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=ec1863dc21e5:36079 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ec1863dc21e5/172.17.0.3:36079 2024-12-07T01:14:27,155 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=ec1863dc21e5,36079,1733534061701, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=ec1863dc21e5:36079 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ec1863dc21e5/172.17.0.3:36079 2024-12-07T01:14:27,155 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=ec1863dc21e5,36079,1733534061701, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=ec1863dc21e5:36079 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ec1863dc21e5/172.17.0.3:36079 2024-12-07T01:14:27,155 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address ec1863dc21e5:36079 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ec1863dc21e5/172.17.0.3:36079 2024-12-07T01:14:27,260 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T01:14:27,260 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ec1863dc21e5,40693,1733534061744, seqNum=-1] 2024-12-07T01:14:27,261 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T01:14:27,262 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59595, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T01:14:27,265 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=80bacdcc81a44a51d622e9d873892ea3, ASSIGN because future has completed 2024-12-07T01:14:27,265 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=72, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 80bacdcc81a44a51d622e9d873892ea3, server=ec1863dc21e5,42915,1733534061580}] 2024-12-07T01:14:27,266 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=324bdaae20264b3880845c3082304e57, ASSIGN because future has completed 2024-12-07T01:14:27,267 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=5d3306d206c48a1c7060e49d4fe00ad5, ASSIGN because future has completed 2024-12-07T01:14:27,268 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=6d8689bfa03ba7aed23b15e5ff5d23ac, ASSIGN because future has completed 2024-12-07T01:14:27,269 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=1cf1ce3ee0822ff957b8ffa13f56df7d, ASSIGN because future has completed 2024-12-07T01:14:27,269 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1cf1ce3ee0822ff957b8ffa13f56df7d, server=ec1863dc21e5,40693,1733534061744}] 2024-12-07T01:14:27,269 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=fa4e20ed49e59ee4cf63b6d84515872f, ASSIGN because future has completed 2024-12-07T01:14:27,270 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=d2b1eb41dda894f4f7b826a2826a9c7b, ASSIGN because future has completed 2024-12-07T01:14:27,271 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=37d131a6ac713be02dc73a29079210f5, ASSIGN because future has completed 2024-12-07T01:14:27,272 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=74, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 37d131a6ac713be02dc73a29079210f5, server=ec1863dc21e5,40693,1733534061744}] 2024-12-07T01:14:27,274 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure d2b1eb41dda894f4f7b826a2826a9c7b, server=ec1863dc21e5,40693,1733534061744}] 2024-12-07T01:14:27,276 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=76, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure fa4e20ed49e59ee4cf63b6d84515872f, server=ec1863dc21e5,40693,1733534061744}] 2024-12-07T01:14:27,279 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6d8689bfa03ba7aed23b15e5ff5d23ac, server=ec1863dc21e5,42915,1733534061580}] 2024-12-07T01:14:27,281 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=78, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5d3306d206c48a1c7060e49d4fe00ad5, server=ec1863dc21e5,42915,1733534061580}] 2024-12-07T01:14:27,284 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 324bdaae20264b3880845c3082304e57, server=ec1863dc21e5,42915,1733534061580}] 2024-12-07T01:14:27,422 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(132): Open test,ooo,1733534064285.324bdaae20264b3880845c3082304e57. 2024-12-07T01:14:27,422 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7752): Opening region: {ENCODED => 324bdaae20264b3880845c3082304e57, NAME => 'test,ooo,1733534064285.324bdaae20264b3880845c3082304e57.', STARTKEY => 'ooo', ENDKEY => 'ppp'} 2024-12-07T01:14:27,423 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 324bdaae20264b3880845c3082304e57 2024-12-07T01:14:27,423 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(898): Instantiated test,ooo,1733534064285.324bdaae20264b3880845c3082304e57.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T01:14:27,423 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7794): checking encryption for 324bdaae20264b3880845c3082304e57 2024-12-07T01:14:27,423 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7797): checking classloading for 324bdaae20264b3880845c3082304e57 2024-12-07T01:14:27,424 INFO [StoreOpener-324bdaae20264b3880845c3082304e57-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 324bdaae20264b3880845c3082304e57 2024-12-07T01:14:27,425 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(132): Open test,lll,1733534064285.d2b1eb41dda894f4f7b826a2826a9c7b. 2024-12-07T01:14:27,425 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7752): Opening region: {ENCODED => d2b1eb41dda894f4f7b826a2826a9c7b, NAME => 'test,lll,1733534064285.d2b1eb41dda894f4f7b826a2826a9c7b.', STARTKEY => 'lll', ENDKEY => 'mmm'} 2024-12-07T01:14:27,425 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test d2b1eb41dda894f4f7b826a2826a9c7b 2024-12-07T01:14:27,425 INFO [StoreOpener-324bdaae20264b3880845c3082304e57-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 324bdaae20264b3880845c3082304e57 columnFamilyName family 2024-12-07T01:14:27,425 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(898): Instantiated test,lll,1733534064285.d2b1eb41dda894f4f7b826a2826a9c7b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T01:14:27,426 DEBUG [StoreOpener-324bdaae20264b3880845c3082304e57-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:14:27,426 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7794): checking encryption for d2b1eb41dda894f4f7b826a2826a9c7b 2024-12-07T01:14:27,426 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7797): checking classloading for d2b1eb41dda894f4f7b826a2826a9c7b 2024-12-07T01:14:27,426 INFO [StoreOpener-324bdaae20264b3880845c3082304e57-1 {}] regionserver.HStore(327): Store=324bdaae20264b3880845c3082304e57/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T01:14:27,426 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1038): replaying wal for 324bdaae20264b3880845c3082304e57 2024-12-07T01:14:27,427 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/default/test/324bdaae20264b3880845c3082304e57 2024-12-07T01:14:27,427 INFO [StoreOpener-d2b1eb41dda894f4f7b826a2826a9c7b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region d2b1eb41dda894f4f7b826a2826a9c7b 2024-12-07T01:14:27,428 INFO [StoreOpener-d2b1eb41dda894f4f7b826a2826a9c7b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d2b1eb41dda894f4f7b826a2826a9c7b columnFamilyName family 2024-12-07T01:14:27,428 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/default/test/324bdaae20264b3880845c3082304e57 2024-12-07T01:14:27,428 DEBUG [StoreOpener-d2b1eb41dda894f4f7b826a2826a9c7b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:14:27,429 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1048): stopping wal replay for 324bdaae20264b3880845c3082304e57 2024-12-07T01:14:27,429 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1060): Cleaning up temporary data for 324bdaae20264b3880845c3082304e57 2024-12-07T01:14:27,429 INFO [StoreOpener-d2b1eb41dda894f4f7b826a2826a9c7b-1 {}] regionserver.HStore(327): Store=d2b1eb41dda894f4f7b826a2826a9c7b/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T01:14:27,429 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1038): replaying wal for d2b1eb41dda894f4f7b826a2826a9c7b 2024-12-07T01:14:27,430 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/default/test/d2b1eb41dda894f4f7b826a2826a9c7b 2024-12-07T01:14:27,431 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1093): writing seq id for 324bdaae20264b3880845c3082304e57 2024-12-07T01:14:27,431 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/default/test/d2b1eb41dda894f4f7b826a2826a9c7b 2024-12-07T01:14:27,431 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1048): stopping wal replay for d2b1eb41dda894f4f7b826a2826a9c7b 2024-12-07T01:14:27,431 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1060): Cleaning up temporary data for d2b1eb41dda894f4f7b826a2826a9c7b 2024-12-07T01:14:27,432 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1114): Opened 324bdaae20264b3880845c3082304e57; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61689920, jitterRate=-0.0807485580444336}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-07T01:14:27,432 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 324bdaae20264b3880845c3082304e57 2024-12-07T01:14:27,432 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1006): Region open journal for 324bdaae20264b3880845c3082304e57: Running coprocessor pre-open hook at 1733534067423Writing region info on filesystem at 1733534067423Initializing all the Stores at 1733534067424 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534067424Cleaning up temporary data from old regions at 1733534067429 (+5 ms)Running coprocessor post-open hooks at 1733534067432 (+3 ms)Region opened successfully at 1733534067432 2024-12-07T01:14:27,433 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1093): writing seq id for d2b1eb41dda894f4f7b826a2826a9c7b 2024-12-07T01:14:27,433 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ooo,1733534064285.324bdaae20264b3880845c3082304e57., pid=79, masterSystemTime=1733534067418 2024-12-07T01:14:27,434 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1114): Opened d2b1eb41dda894f4f7b826a2826a9c7b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62260324, jitterRate=-0.07224887609481812}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-07T01:14:27,434 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d2b1eb41dda894f4f7b826a2826a9c7b 2024-12-07T01:14:27,434 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1006): Region open journal for d2b1eb41dda894f4f7b826a2826a9c7b: Running coprocessor pre-open hook at 1733534067426Writing region info on filesystem at 1733534067426Initializing all the Stores at 1733534067427 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534067427Cleaning up temporary data from old regions at 1733534067431 (+4 ms)Running coprocessor post-open hooks at 1733534067434 (+3 ms)Region opened successfully at 1733534067434 2024-12-07T01:14:27,435 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2236): Post open deploy tasks for test,lll,1733534064285.d2b1eb41dda894f4f7b826a2826a9c7b., pid=75, masterSystemTime=1733534067421 2024-12-07T01:14:27,436 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ooo,1733534064285.324bdaae20264b3880845c3082304e57. 2024-12-07T01:14:27,436 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(153): Opened test,ooo,1733534064285.324bdaae20264b3880845c3082304e57. 2024-12-07T01:14:27,436 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(132): Open test,ccc,1733534064285.6d8689bfa03ba7aed23b15e5ff5d23ac. 2024-12-07T01:14:27,436 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7752): Opening region: {ENCODED => 6d8689bfa03ba7aed23b15e5ff5d23ac, NAME => 'test,ccc,1733534064285.6d8689bfa03ba7aed23b15e5ff5d23ac.', STARTKEY => 'ccc', ENDKEY => 'ddd'} 2024-12-07T01:14:27,437 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 6d8689bfa03ba7aed23b15e5ff5d23ac 2024-12-07T01:14:27,437 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(898): Instantiated test,ccc,1733534064285.6d8689bfa03ba7aed23b15e5ff5d23ac.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T01:14:27,437 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7794): checking encryption for 6d8689bfa03ba7aed23b15e5ff5d23ac 2024-12-07T01:14:27,437 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7797): checking classloading for 6d8689bfa03ba7aed23b15e5ff5d23ac 2024-12-07T01:14:27,437 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2266): Finished post open deploy task for test,lll,1733534064285.d2b1eb41dda894f4f7b826a2826a9c7b. 2024-12-07T01:14:27,438 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(153): Opened test,lll,1733534064285.d2b1eb41dda894f4f7b826a2826a9c7b. 2024-12-07T01:14:27,438 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] handler.AssignRegionHandler(132): Open test,fff,1733534064285.37d131a6ac713be02dc73a29079210f5. 2024-12-07T01:14:27,438 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(7752): Opening region: {ENCODED => 37d131a6ac713be02dc73a29079210f5, NAME => 'test,fff,1733534064285.37d131a6ac713be02dc73a29079210f5.', STARTKEY => 'fff', ENDKEY => 'ggg'} 2024-12-07T01:14:27,438 WARN [PEWorker-1 {}] procedure2.ProcedureTestingUtility(201): Set Kill before store update in rollback to: true 2024-12-07T01:14:27,438 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 37d131a6ac713be02dc73a29079210f5 2024-12-07T01:14:27,438 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=79, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 324bdaae20264b3880845c3082304e57, server=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:27,438 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(898): Instantiated test,fff,1733534064285.37d131a6ac713be02dc73a29079210f5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T01:14:27,438 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(7794): checking encryption for 37d131a6ac713be02dc73a29079210f5 2024-12-07T01:14:27,438 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(7797): checking classloading for 37d131a6ac713be02dc73a29079210f5 2024-12-07T01:14:27,439 INFO [StoreOpener-6d8689bfa03ba7aed23b15e5ff5d23ac-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 6d8689bfa03ba7aed23b15e5ff5d23ac 2024-12-07T01:14:27,438 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1844): CODE-BUG: Uncaught runtime exception: pid=79, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 324bdaae20264b3880845c3082304e57, server=ec1863dc21e5,42915,1733534061580 java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-07T01:14:27,440 INFO [StoreOpener-6d8689bfa03ba7aed23b15e5ff5d23ac-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6d8689bfa03ba7aed23b15e5ff5d23ac columnFamilyName family 2024-12-07T01:14:27,440 DEBUG [StoreOpener-6d8689bfa03ba7aed23b15e5ff5d23ac-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:14:27,440 INFO [StoreOpener-37d131a6ac713be02dc73a29079210f5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 37d131a6ac713be02dc73a29079210f5 2024-12-07T01:14:27,440 INFO [StoreOpener-6d8689bfa03ba7aed23b15e5ff5d23ac-1 {}] regionserver.HStore(327): Store=6d8689bfa03ba7aed23b15e5ff5d23ac/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T01:14:27,440 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1038): replaying wal for 6d8689bfa03ba7aed23b15e5ff5d23ac 2024-12-07T01:14:27,441 INFO [StoreOpener-37d131a6ac713be02dc73a29079210f5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 37d131a6ac713be02dc73a29079210f5 columnFamilyName family 2024-12-07T01:14:27,441 DEBUG [StoreOpener-37d131a6ac713be02dc73a29079210f5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:14:27,441 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/default/test/6d8689bfa03ba7aed23b15e5ff5d23ac 2024-12-07T01:14:27,441 INFO [StoreOpener-37d131a6ac713be02dc73a29079210f5-1 {}] regionserver.HStore(327): Store=37d131a6ac713be02dc73a29079210f5/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T01:14:27,442 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1038): replaying wal for 37d131a6ac713be02dc73a29079210f5 2024-12-07T01:14:27,443 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/default/test/37d131a6ac713be02dc73a29079210f5 2024-12-07T01:14:27,443 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/default/test/6d8689bfa03ba7aed23b15e5ff5d23ac 2024-12-07T01:14:27,443 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1048): stopping wal replay for 6d8689bfa03ba7aed23b15e5ff5d23ac 2024-12-07T01:14:27,443 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1060): Cleaning up temporary data for 6d8689bfa03ba7aed23b15e5ff5d23ac 2024-12-07T01:14:27,444 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/default/test/37d131a6ac713be02dc73a29079210f5 2024-12-07T01:14:27,445 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1048): stopping wal replay for 37d131a6ac713be02dc73a29079210f5 2024-12-07T01:14:27,445 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1060): Cleaning up temporary data for 37d131a6ac713be02dc73a29079210f5 2024-12-07T01:14:27,445 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1093): writing seq id for 6d8689bfa03ba7aed23b15e5ff5d23ac 2024-12-07T01:14:27,446 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1114): Opened 6d8689bfa03ba7aed23b15e5ff5d23ac; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70486046, jitterRate=0.050323933362960815}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-07T01:14:27,446 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6d8689bfa03ba7aed23b15e5ff5d23ac 2024-12-07T01:14:27,446 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1006): Region open journal for 6d8689bfa03ba7aed23b15e5ff5d23ac: Running coprocessor pre-open hook at 1733534067437Writing region info on filesystem at 1733534067437Initializing all the Stores at 1733534067438 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534067438Cleaning up temporary data from old regions at 1733534067443 (+5 ms)Running coprocessor post-open hooks at 1733534067446 (+3 ms)Region opened successfully at 1733534067446 2024-12-07T01:14:27,447 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1093): writing seq id for 37d131a6ac713be02dc73a29079210f5 2024-12-07T01:14:27,447 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ccc,1733534064285.6d8689bfa03ba7aed23b15e5ff5d23ac., pid=77, masterSystemTime=1733534067418 2024-12-07T01:14:27,448 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1114): Opened 37d131a6ac713be02dc73a29079210f5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63190063, jitterRate=-0.05839468538761139}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-07T01:14:27,448 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 37d131a6ac713be02dc73a29079210f5 2024-12-07T01:14:27,448 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1006): Region open journal for 37d131a6ac713be02dc73a29079210f5: Running coprocessor pre-open hook at 1733534067439Writing region info on filesystem at 1733534067439Initializing all the Stores at 1733534067439Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534067440 (+1 ms)Cleaning up temporary data from old regions at 1733534067445 (+5 ms)Running coprocessor post-open hooks at 1733534067448 (+3 ms)Region opened successfully at 1733534067448 2024-12-07T01:14:27,449 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegionServer(2236): Post open deploy tasks for test,fff,1733534064285.37d131a6ac713be02dc73a29079210f5., pid=74, masterSystemTime=1733534067421 2024-12-07T01:14:27,449 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=true, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 324bdaae20264b3880845c3082304e57, server=ec1863dc21e5,42915,1733534061580:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ec1863dc21e5,36079,1733534061701, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1845) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 4 more 2024-12-07T01:14:27,450 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ccc,1733534064285.6d8689bfa03ba7aed23b15e5ff5d23ac. 2024-12-07T01:14:27,450 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(153): Opened test,ccc,1733534064285.6d8689bfa03ba7aed23b15e5ff5d23ac. 2024-12-07T01:14:27,450 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] handler.AssignRegionHandler(132): Open test,uuu,1733534064285.80bacdcc81a44a51d622e9d873892ea3. 2024-12-07T01:14:27,450 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(7752): Opening region: {ENCODED => 80bacdcc81a44a51d622e9d873892ea3, NAME => 'test,uuu,1733534064285.80bacdcc81a44a51d622e9d873892ea3.', STARTKEY => 'uuu', ENDKEY => 'vvv'} 2024-12-07T01:14:27,450 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-07T01:14:27,450 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=79, ppid=65, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 324bdaae20264b3880845c3082304e57, server=ec1863dc21e5,42915,1733534061580:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 324bdaae20264b3880845c3082304e57, server=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:27,450 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-07T01:14:27,450 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 80bacdcc81a44a51d622e9d873892ea3 2024-12-07T01:14:27,450 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(898): Instantiated test,uuu,1733534064285.80bacdcc81a44a51d622e9d873892ea3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T01:14:27,451 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(7794): checking encryption for 80bacdcc81a44a51d622e9d873892ea3 2024-12-07T01:14:27,451 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(7797): checking classloading for 80bacdcc81a44a51d622e9d873892ea3 2024-12-07T01:14:27,451 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=79, ppid=65, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 324bdaae20264b3880845c3082304e57, server=ec1863dc21e5,42915,1733534061580:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 324bdaae20264b3880845c3082304e57, server=ec1863dc21e5,42915,1733534061580 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-07T01:14:27,452 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegionServer(2266): Finished post open deploy task for test,fff,1733534064285.37d131a6ac713be02dc73a29079210f5. 2024-12-07T01:14:27,452 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] handler.AssignRegionHandler(153): Opened test,fff,1733534064285.37d131a6ac713be02dc73a29079210f5. 2024-12-07T01:14:27,452 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] handler.AssignRegionHandler(132): Open test,xxx,1733534064285.1cf1ce3ee0822ff957b8ffa13f56df7d. 2024-12-07T01:14:27,452 INFO [StoreOpener-80bacdcc81a44a51d622e9d873892ea3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 80bacdcc81a44a51d622e9d873892ea3 2024-12-07T01:14:27,453 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(7752): Opening region: {ENCODED => 1cf1ce3ee0822ff957b8ffa13f56df7d, NAME => 'test,xxx,1733534064285.1cf1ce3ee0822ff957b8ffa13f56df7d.', STARTKEY => 'xxx', ENDKEY => 'yyy'} 2024-12-07T01:14:27,453 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 1cf1ce3ee0822ff957b8ffa13f56df7d 2024-12-07T01:14:27,453 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(898): Instantiated test,xxx,1733534064285.1cf1ce3ee0822ff957b8ffa13f56df7d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T01:14:27,453 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(7794): checking encryption for 1cf1ce3ee0822ff957b8ffa13f56df7d 2024-12-07T01:14:27,453 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(7797): checking classloading for 1cf1ce3ee0822ff957b8ffa13f56df7d 2024-12-07T01:14:27,454 INFO [StoreOpener-80bacdcc81a44a51d622e9d873892ea3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 80bacdcc81a44a51d622e9d873892ea3 columnFamilyName family 2024-12-07T01:14:27,454 DEBUG [StoreOpener-80bacdcc81a44a51d622e9d873892ea3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:14:27,454 INFO [StoreOpener-80bacdcc81a44a51d622e9d873892ea3-1 {}] regionserver.HStore(327): Store=80bacdcc81a44a51d622e9d873892ea3/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T01:14:27,454 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1038): replaying wal for 80bacdcc81a44a51d622e9d873892ea3 2024-12-07T01:14:27,455 INFO [StoreOpener-1cf1ce3ee0822ff957b8ffa13f56df7d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 1cf1ce3ee0822ff957b8ffa13f56df7d 2024-12-07T01:14:27,455 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/default/test/80bacdcc81a44a51d622e9d873892ea3 2024-12-07T01:14:27,456 INFO [StoreOpener-1cf1ce3ee0822ff957b8ffa13f56df7d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1cf1ce3ee0822ff957b8ffa13f56df7d columnFamilyName family 2024-12-07T01:14:27,456 DEBUG [StoreOpener-1cf1ce3ee0822ff957b8ffa13f56df7d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:14:27,456 INFO [StoreOpener-1cf1ce3ee0822ff957b8ffa13f56df7d-1 {}] regionserver.HStore(327): Store=1cf1ce3ee0822ff957b8ffa13f56df7d/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T01:14:27,456 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1038): replaying wal for 1cf1ce3ee0822ff957b8ffa13f56df7d 2024-12-07T01:14:27,456 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/default/test/80bacdcc81a44a51d622e9d873892ea3 2024-12-07T01:14:27,457 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1048): stopping wal replay for 80bacdcc81a44a51d622e9d873892ea3 2024-12-07T01:14:27,457 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1060): Cleaning up temporary data for 80bacdcc81a44a51d622e9d873892ea3 2024-12-07T01:14:27,457 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/default/test/1cf1ce3ee0822ff957b8ffa13f56df7d 2024-12-07T01:14:27,458 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/default/test/1cf1ce3ee0822ff957b8ffa13f56df7d 2024-12-07T01:14:27,459 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1048): stopping wal replay for 1cf1ce3ee0822ff957b8ffa13f56df7d 2024-12-07T01:14:27,459 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1060): Cleaning up temporary data for 1cf1ce3ee0822ff957b8ffa13f56df7d 2024-12-07T01:14:27,459 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1093): writing seq id for 80bacdcc81a44a51d622e9d873892ea3 2024-12-07T01:14:27,460 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1114): Opened 80bacdcc81a44a51d622e9d873892ea3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71464769, jitterRate=0.06490804255008698}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-07T01:14:27,460 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 80bacdcc81a44a51d622e9d873892ea3 2024-12-07T01:14:27,460 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1006): Region open journal for 80bacdcc81a44a51d622e9d873892ea3: Running coprocessor pre-open hook at 1733534067451Writing region info on filesystem at 1733534067451Initializing all the Stores at 1733534067452 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534067452Cleaning up temporary data from old regions at 1733534067457 (+5 ms)Running coprocessor post-open hooks at 1733534067460 (+3 ms)Region opened successfully at 1733534067460 2024-12-07T01:14:27,461 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1093): writing seq id for 1cf1ce3ee0822ff957b8ffa13f56df7d 2024-12-07T01:14:27,462 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1114): Opened 1cf1ce3ee0822ff957b8ffa13f56df7d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73951654, jitterRate=0.1019655168056488}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-07T01:14:27,462 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegionServer(2236): Post open deploy tasks for test,uuu,1733534064285.80bacdcc81a44a51d622e9d873892ea3., pid=72, masterSystemTime=1733534067418 2024-12-07T01:14:27,462 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1cf1ce3ee0822ff957b8ffa13f56df7d 2024-12-07T01:14:27,462 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1006): Region open journal for 1cf1ce3ee0822ff957b8ffa13f56df7d: Running coprocessor pre-open hook at 1733534067453Writing region info on filesystem at 1733534067453Initializing all the Stores at 1733534067454 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534067454Cleaning up temporary data from old regions at 1733534067459 (+5 ms)Running coprocessor post-open hooks at 1733534067462 (+3 ms)Region opened successfully at 1733534067462 2024-12-07T01:14:27,463 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegionServer(2236): Post open deploy tasks for test,xxx,1733534064285.1cf1ce3ee0822ff957b8ffa13f56df7d., pid=73, masterSystemTime=1733534067421 2024-12-07T01:14:27,464 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegionServer(2266): Finished post open deploy task for test,uuu,1733534064285.80bacdcc81a44a51d622e9d873892ea3. 2024-12-07T01:14:27,464 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] handler.AssignRegionHandler(153): Opened test,uuu,1733534064285.80bacdcc81a44a51d622e9d873892ea3. 2024-12-07T01:14:27,464 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] handler.AssignRegionHandler(132): Open test,iii,1733534064285.5d3306d206c48a1c7060e49d4fe00ad5. 2024-12-07T01:14:27,464 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7752): Opening region: {ENCODED => 5d3306d206c48a1c7060e49d4fe00ad5, NAME => 'test,iii,1733534064285.5d3306d206c48a1c7060e49d4fe00ad5.', STARTKEY => 'iii', ENDKEY => 'jjj'} 2024-12-07T01:14:27,465 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 5d3306d206c48a1c7060e49d4fe00ad5 2024-12-07T01:14:27,465 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(898): Instantiated test,iii,1733534064285.5d3306d206c48a1c7060e49d4fe00ad5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T01:14:27,465 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7794): checking encryption for 5d3306d206c48a1c7060e49d4fe00ad5 2024-12-07T01:14:27,465 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7797): checking classloading for 5d3306d206c48a1c7060e49d4fe00ad5 2024-12-07T01:14:27,465 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegionServer(2266): Finished post open deploy task for test,xxx,1733534064285.1cf1ce3ee0822ff957b8ffa13f56df7d. 2024-12-07T01:14:27,465 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] handler.AssignRegionHandler(153): Opened test,xxx,1733534064285.1cf1ce3ee0822ff957b8ffa13f56df7d. 2024-12-07T01:14:27,465 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] handler.AssignRegionHandler(132): Open test,rrr,1733534064285.fa4e20ed49e59ee4cf63b6d84515872f. 2024-12-07T01:14:27,465 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7752): Opening region: {ENCODED => fa4e20ed49e59ee4cf63b6d84515872f, NAME => 'test,rrr,1733534064285.fa4e20ed49e59ee4cf63b6d84515872f.', STARTKEY => 'rrr', ENDKEY => 'sss'} 2024-12-07T01:14:27,465 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test fa4e20ed49e59ee4cf63b6d84515872f 2024-12-07T01:14:27,466 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(898): Instantiated test,rrr,1733534064285.fa4e20ed49e59ee4cf63b6d84515872f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T01:14:27,466 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7794): checking encryption for fa4e20ed49e59ee4cf63b6d84515872f 2024-12-07T01:14:27,466 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7797): checking classloading for fa4e20ed49e59ee4cf63b6d84515872f 2024-12-07T01:14:27,467 INFO [StoreOpener-5d3306d206c48a1c7060e49d4fe00ad5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 5d3306d206c48a1c7060e49d4fe00ad5 2024-12-07T01:14:27,468 INFO [StoreOpener-fa4e20ed49e59ee4cf63b6d84515872f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region fa4e20ed49e59ee4cf63b6d84515872f 2024-12-07T01:14:27,468 INFO [StoreOpener-5d3306d206c48a1c7060e49d4fe00ad5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5d3306d206c48a1c7060e49d4fe00ad5 columnFamilyName family 2024-12-07T01:14:27,468 DEBUG [StoreOpener-5d3306d206c48a1c7060e49d4fe00ad5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:14:27,468 INFO [StoreOpener-fa4e20ed49e59ee4cf63b6d84515872f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fa4e20ed49e59ee4cf63b6d84515872f columnFamilyName family 2024-12-07T01:14:27,468 INFO [StoreOpener-5d3306d206c48a1c7060e49d4fe00ad5-1 {}] regionserver.HStore(327): Store=5d3306d206c48a1c7060e49d4fe00ad5/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T01:14:27,468 DEBUG [StoreOpener-fa4e20ed49e59ee4cf63b6d84515872f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:14:27,469 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1038): replaying wal for 5d3306d206c48a1c7060e49d4fe00ad5 2024-12-07T01:14:27,469 INFO [StoreOpener-fa4e20ed49e59ee4cf63b6d84515872f-1 {}] regionserver.HStore(327): Store=fa4e20ed49e59ee4cf63b6d84515872f/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T01:14:27,469 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1038): replaying wal for fa4e20ed49e59ee4cf63b6d84515872f 2024-12-07T01:14:27,469 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/default/test/5d3306d206c48a1c7060e49d4fe00ad5 2024-12-07T01:14:27,470 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/default/test/fa4e20ed49e59ee4cf63b6d84515872f 2024-12-07T01:14:27,470 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/default/test/5d3306d206c48a1c7060e49d4fe00ad5 2024-12-07T01:14:27,471 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/default/test/fa4e20ed49e59ee4cf63b6d84515872f 2024-12-07T01:14:27,471 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1048): stopping wal replay for 5d3306d206c48a1c7060e49d4fe00ad5 2024-12-07T01:14:27,471 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1060): Cleaning up temporary data for 5d3306d206c48a1c7060e49d4fe00ad5 2024-12-07T01:14:27,471 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1048): stopping wal replay for fa4e20ed49e59ee4cf63b6d84515872f 2024-12-07T01:14:27,471 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1060): Cleaning up temporary data for fa4e20ed49e59ee4cf63b6d84515872f 2024-12-07T01:14:27,473 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1093): writing seq id for 5d3306d206c48a1c7060e49d4fe00ad5 2024-12-07T01:14:27,473 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1093): writing seq id for fa4e20ed49e59ee4cf63b6d84515872f 2024-12-07T01:14:27,474 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1114): Opened fa4e20ed49e59ee4cf63b6d84515872f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60157007, jitterRate=-0.10359074175357819}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-07T01:14:27,474 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fa4e20ed49e59ee4cf63b6d84515872f 2024-12-07T01:14:27,474 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1114): Opened 5d3306d206c48a1c7060e49d4fe00ad5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74260814, jitterRate=0.10657235980033875}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-07T01:14:27,474 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1006): Region open journal for fa4e20ed49e59ee4cf63b6d84515872f: Running coprocessor pre-open hook at 1733534067466Writing region info on filesystem at 1733534067466Initializing all the Stores at 1733534067467 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534067467Cleaning up temporary data from old regions at 1733534067471 (+4 ms)Running coprocessor post-open hooks at 1733534067474 (+3 ms)Region opened successfully at 1733534067474 2024-12-07T01:14:27,474 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5d3306d206c48a1c7060e49d4fe00ad5 2024-12-07T01:14:27,474 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1006): Region open journal for 5d3306d206c48a1c7060e49d4fe00ad5: Running coprocessor pre-open hook at 1733534067465Writing region info on filesystem at 1733534067465Initializing all the Stores at 1733534067466 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534067466Cleaning up temporary data from old regions at 1733534067471 (+5 ms)Running coprocessor post-open hooks at 1733534067474 (+3 ms)Region opened successfully at 1733534067474 2024-12-07T01:14:27,475 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionServer(2236): Post open deploy tasks for test,iii,1733534064285.5d3306d206c48a1c7060e49d4fe00ad5., pid=78, masterSystemTime=1733534067418 2024-12-07T01:14:27,475 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2236): Post open deploy tasks for test,rrr,1733534064285.fa4e20ed49e59ee4cf63b6d84515872f., pid=76, masterSystemTime=1733534067421 2024-12-07T01:14:27,477 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2266): Finished post open deploy task for test,rrr,1733534064285.fa4e20ed49e59ee4cf63b6d84515872f. 2024-12-07T01:14:27,477 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] handler.AssignRegionHandler(153): Opened test,rrr,1733534064285.fa4e20ed49e59ee4cf63b6d84515872f. 2024-12-07T01:14:27,478 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionServer(2266): Finished post open deploy task for test,iii,1733534064285.5d3306d206c48a1c7060e49d4fe00ad5. 2024-12-07T01:14:27,478 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] handler.AssignRegionHandler(153): Opened test,iii,1733534064285.5d3306d206c48a1c7060e49d4fe00ad5. 2024-12-07T01:14:27,544 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-07T01:14:27,545 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-07T01:14:27,545 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-07T01:14:27,545 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-07T01:14:27,546 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-07T01:14:27,546 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-07T01:14:27,546 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-07T01:14:27,546 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-07T01:14:27,547 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-07T01:14:27,553 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-07T01:14:27,553 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T01:14:27,554 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=64, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-07T01:14:27,554 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T01:14:27,554 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-07T01:14:27,554 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T01:14:27,554 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-07T01:14:27,554 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T01:14:27,554 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-07T01:14:27,554 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T01:14:27,554 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-07T01:14:27,554 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T01:14:27,554 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-07T01:14:27,554 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T01:14:27,554 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-07T01:14:27,554 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T01:14:27,554 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-07T01:14:27,554 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-07T01:14:27,554 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-07T01:14:27,554 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T01:14:27,554 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-07T01:14:27,554 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-07T01:14:27,554 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-07T01:14:27,555 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-07T01:14:27,556 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-07T01:14:27,561 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 324bdaae20264b3880845c3082304e57, server=ec1863dc21e5,42915,1733534061580:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ec1863dc21e5,36079,1733534061701, splitWal=true, meta=true 2024-12-07T01:14:27,561 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ec1863dc21e5%2C36079%2C1733534061701.meta.1733534063927.meta, worker=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:27,561 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ec1863dc21e5%2C36079%2C1733534061701.meta.1733534063927.meta, worker=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:27,561 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-07T01:14:27,561 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ec1863dc21e5,40693,1733534061744 2024-12-07T01:14:27,561 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ec1863dc21e5%2C36079%2C1733534061701.1733534063448, worker=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:27,561 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ec1863dc21e5%2C36079%2C1733534061701.1733534063448, worker=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:27,561 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=37d131a6ac713be02dc73a29079210f5, ASSIGN 2024-12-07T01:14:27,561 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=324bdaae20264b3880845c3082304e57, ASSIGN 2024-12-07T01:14:27,561 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1cf1ce3ee0822ff957b8ffa13f56df7d, ASSIGN 2024-12-07T01:14:27,562 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6d8689bfa03ba7aed23b15e5ff5d23ac, ASSIGN 2024-12-07T01:14:27,562 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5d3306d206c48a1c7060e49d4fe00ad5, ASSIGN 2024-12-07T01:14:27,562 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=80bacdcc81a44a51d622e9d873892ea3, ASSIGN 2024-12-07T01:14:27,562 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d2b1eb41dda894f4f7b826a2826a9c7b, ASSIGN 2024-12-07T01:14:27,562 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fa4e20ed49e59ee4cf63b6d84515872f, ASSIGN 2024-12-07T01:14:27,562 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 80bacdcc81a44a51d622e9d873892ea3, server=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:27,562 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1cf1ce3ee0822ff957b8ffa13f56df7d, server=ec1863dc21e5,40693,1733534061744 2024-12-07T01:14:27,562 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 37d131a6ac713be02dc73a29079210f5, server=ec1863dc21e5,40693,1733534061744 2024-12-07T01:14:27,562 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure d2b1eb41dda894f4f7b826a2826a9c7b, server=ec1863dc21e5,40693,1733534061744 2024-12-07T01:14:27,562 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure fa4e20ed49e59ee4cf63b6d84515872f, server=ec1863dc21e5,40693,1733534061744 2024-12-07T01:14:27,562 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6d8689bfa03ba7aed23b15e5ff5d23ac, server=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:27,562 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5d3306d206c48a1c7060e49d4fe00ad5, server=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:27,562 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=65, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 324bdaae20264b3880845c3082304e57, server=ec1863dc21e5,42915,1733534061580:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 324bdaae20264b3880845c3082304e57, server=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:27,562 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 324bdaae20264b3880845c3082304e57, server=ec1863dc21e5,42915,1733534061580:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ec1863dc21e5,36079,1733534061701, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:27,562 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ec1863dc21e5%2C36079%2C1733534061701.meta.1733534063927.meta, worker=ec1863dc21e5,42915,1733534061580 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:27,562 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ec1863dc21e5%2C36079%2C1733534061701.meta.1733534063927.meta, worker=ec1863dc21e5,42915,1733534061580 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:27,562 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:27,562 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ec1863dc21e5,40693,1733534061744 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:27,563 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ec1863dc21e5%2C36079%2C1733534061701.1733534063448, worker=ec1863dc21e5,42915,1733534061580 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:27,563 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ec1863dc21e5%2C36079%2C1733534061701.1733534063448, worker=ec1863dc21e5,42915,1733534061580 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:27,563 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=37d131a6ac713be02dc73a29079210f5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:27,563 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=37d131a6ac713be02dc73a29079210f5, ASSIGN 2024-12-07T01:14:27,563 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=324bdaae20264b3880845c3082304e57, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:27,563 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=324bdaae20264b3880845c3082304e57, ASSIGN 2024-12-07T01:14:27,563 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1cf1ce3ee0822ff957b8ffa13f56df7d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:27,563 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1cf1ce3ee0822ff957b8ffa13f56df7d, ASSIGN 2024-12-07T01:14:27,563 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6d8689bfa03ba7aed23b15e5ff5d23ac, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:27,563 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6d8689bfa03ba7aed23b15e5ff5d23ac, ASSIGN 2024-12-07T01:14:27,563 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5d3306d206c48a1c7060e49d4fe00ad5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:27,563 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5d3306d206c48a1c7060e49d4fe00ad5, ASSIGN 2024-12-07T01:14:27,563 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=80bacdcc81a44a51d622e9d873892ea3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:27,563 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=80bacdcc81a44a51d622e9d873892ea3, ASSIGN 2024-12-07T01:14:27,563 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d2b1eb41dda894f4f7b826a2826a9c7b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:27,563 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d2b1eb41dda894f4f7b826a2826a9c7b, ASSIGN 2024-12-07T01:14:27,563 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fa4e20ed49e59ee4cf63b6d84515872f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:27,563 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fa4e20ed49e59ee4cf63b6d84515872f, ASSIGN 2024-12-07T01:14:27,563 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 80bacdcc81a44a51d622e9d873892ea3, server=ec1863dc21e5,42915,1733534061580 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:27,563 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1cf1ce3ee0822ff957b8ffa13f56df7d, server=ec1863dc21e5,40693,1733534061744 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:27,563 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 37d131a6ac713be02dc73a29079210f5, server=ec1863dc21e5,40693,1733534061744 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:27,564 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure d2b1eb41dda894f4f7b826a2826a9c7b, server=ec1863dc21e5,40693,1733534061744 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:27,564 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure fa4e20ed49e59ee4cf63b6d84515872f, server=ec1863dc21e5,40693,1733534061744 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:27,564 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6d8689bfa03ba7aed23b15e5ff5d23ac, server=ec1863dc21e5,42915,1733534061580 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:27,564 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5d3306d206c48a1c7060e49d4fe00ad5, server=ec1863dc21e5,42915,1733534061580 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:27,564 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=65, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 324bdaae20264b3880845c3082304e57, server=ec1863dc21e5,42915,1733534061580:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 324bdaae20264b3880845c3082304e57, server=ec1863dc21e5,42915,1733534061580 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:27,564 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 17 msec 2024-12-07T01:14:27,565 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46141 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T01:14:27,566 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=61 2024-12-07T01:14:27,566 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=ec1863dc21e5,40693,1733534061744, table=hbase:meta, region=1588230740 2024-12-07T01:14:27,566 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ec1863dc21e5,40693,1733534061744, state=OPEN 2024-12-07T01:14:27,591 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46141 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T01:14:27,654 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40693-0x101ad5a899b0003, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T01:14:27,654 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42915-0x101ad5a899b0001, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T01:14:27,654 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46141-0x101ad5a899b0000, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T01:14:27,654 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T01:14:27,654 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T01:14:27,654 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T01:14:27,655 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=37d131a6ac713be02dc73a29079210f5, ASSIGN to state=OFFLINE, location=null, table=test, region=37d131a6ac713be02dc73a29079210f5 to restore RIT 2024-12-07T01:14:27,655 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=324bdaae20264b3880845c3082304e57, ASSIGN to state=OFFLINE, location=null, table=test, region=324bdaae20264b3880845c3082304e57 to restore RIT 2024-12-07T01:14:27,656 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1cf1ce3ee0822ff957b8ffa13f56df7d, ASSIGN to state=OFFLINE, location=null, table=test, region=1cf1ce3ee0822ff957b8ffa13f56df7d to restore RIT 2024-12-07T01:14:27,656 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6d8689bfa03ba7aed23b15e5ff5d23ac, ASSIGN to state=OFFLINE, location=null, table=test, region=6d8689bfa03ba7aed23b15e5ff5d23ac to restore RIT 2024-12-07T01:14:27,656 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5d3306d206c48a1c7060e49d4fe00ad5, ASSIGN to state=OFFLINE, location=null, table=test, region=5d3306d206c48a1c7060e49d4fe00ad5 to restore RIT 2024-12-07T01:14:27,656 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=80bacdcc81a44a51d622e9d873892ea3, ASSIGN to state=OFFLINE, location=null, table=test, region=80bacdcc81a44a51d622e9d873892ea3 to restore RIT 2024-12-07T01:14:27,656 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d2b1eb41dda894f4f7b826a2826a9c7b, ASSIGN to state=OFFLINE, location=null, table=test, region=d2b1eb41dda894f4f7b826a2826a9c7b to restore RIT 2024-12-07T01:14:27,656 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fa4e20ed49e59ee4cf63b6d84515872f, ASSIGN to state=OFFLINE, location=null, table=test, region=fa4e20ed49e59ee4cf63b6d84515872f to restore RIT 2024-12-07T01:14:27,656 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733534097656 2024-12-07T01:14:27,656 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-07T01:14:27,656 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 324bdaae20264b3880845c3082304e57, server=ec1863dc21e5,42915,1733534061580:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ec1863dc21e5,36079,1733534061701, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-07T01:14:27,658 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-07T01:14:27,659 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-07T01:14:27,659 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=78, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5d3306d206c48a1c7060e49d4fe00ad5, server=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:27,659 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-07T01:14:27,659 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=78, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5d3306d206c48a1c7060e49d4fe00ad5, server=ec1863dc21e5,42915,1733534061580 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-07T01:14:27,681 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=976bbe8d7afc3b1e02af11d1b515e82c, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,681 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0980fd6969c12f09b843d772c626dfa5, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,681 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d8689bfa03ba7aed23b15e5ff5d23ac, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,682 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=af843c9df2505d2eeee3c18062e9a2a6, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,682 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c0a570532487ea0ebbc51cbec9ff7e37, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,682 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=37d131a6ac713be02dc73a29079210f5, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,682 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c0f43810347933724c9e6319b11ae5cc, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,683 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9c426833151f173ef0d81d9c0ae11767, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,683 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5d3306d206c48a1c7060e49d4fe00ad5, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,683 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aa3dca7b6c799eb77565fb0e20ef1c15, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,683 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=75c927364905f2c6bffff16328d8aaf5, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,684 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d2b1eb41dda894f4f7b826a2826a9c7b, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,684 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=327d2bdcc20803fcfd1e61553f30068b, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,684 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3fe1b6a546d269c366ae3f13263e1a2a, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,684 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=324bdaae20264b3880845c3082304e57, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,684 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c66f402dd9e22edaa5996a3b7446808c, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,685 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=856730db89c49e65ccb40b5967ccf99d, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,685 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fa4e20ed49e59ee4cf63b6d84515872f, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,685 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1da36e78b320d37341113b4185c91b26, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,685 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3f148a9f8c5bea63f4744afb36553ef7, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,685 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=80bacdcc81a44a51d622e9d873892ea3, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,686 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=76cd5c1764feda4c992319b218572cef, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,686 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8cd2040dc039667d68bc1a24f5f2322f, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,686 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1cf1ce3ee0822ff957b8ffa13f56df7d, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,686 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b59f40eb016096571b13ae445f262449, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,686 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=73ec7ce1c93542e31d99b7b4474a9750, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,686 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-07T01:14:27,686 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733534127686 2024-12-07T01:14:27,686 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733534187686 2024-12-07T01:14:27,686 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 30 msec 2024-12-07T01:14:27,687 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-07T01:14:27,687 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-07T01:14:27,687 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-07T01:14:27,687 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-07T01:14:27,687 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-07T01:14:27,687 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-07T01:14:27,687 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-07T01:14:27,687 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-07T01:14:27,688 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-07T01:14:27,691 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-07T01:14:27,692 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T01:14:27,692 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=64, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-07T01:14:27,692 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T01:14:27,692 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T01:14:27,692 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-07T01:14:27,692 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T01:14:27,692 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-07T01:14:27,692 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T01:14:27,692 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-07T01:14:27,692 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T01:14:27,692 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-07T01:14:27,692 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T01:14:27,692 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-07T01:14:27,692 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T01:14:27,692 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-07T01:14:27,692 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-07T01:14:27,692 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-07T01:14:27,692 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T01:14:27,692 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-07T01:14:27,692 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-07T01:14:27,692 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-07T01:14:27,692 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-07T01:14:27,693 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-07T01:14:27,695 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 324bdaae20264b3880845c3082304e57, server=ec1863dc21e5,42915,1733534061580:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ec1863dc21e5,36079,1733534061701, splitWal=true, meta=true 2024-12-07T01:14:27,696 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ec1863dc21e5%2C36079%2C1733534061701.meta.1733534063927.meta, worker=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:27,696 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ec1863dc21e5%2C36079%2C1733534061701.meta.1733534063927.meta, worker=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:27,696 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-07T01:14:27,696 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ec1863dc21e5,40693,1733534061744 2024-12-07T01:14:27,696 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ec1863dc21e5%2C36079%2C1733534061701.1733534063448, worker=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:27,696 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ec1863dc21e5%2C36079%2C1733534061701.1733534063448, worker=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:27,696 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=37d131a6ac713be02dc73a29079210f5, ASSIGN 2024-12-07T01:14:27,696 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=324bdaae20264b3880845c3082304e57, ASSIGN 2024-12-07T01:14:27,696 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1cf1ce3ee0822ff957b8ffa13f56df7d, ASSIGN 2024-12-07T01:14:27,696 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6d8689bfa03ba7aed23b15e5ff5d23ac, ASSIGN 2024-12-07T01:14:27,696 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5d3306d206c48a1c7060e49d4fe00ad5, ASSIGN 2024-12-07T01:14:27,696 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=80bacdcc81a44a51d622e9d873892ea3, ASSIGN 2024-12-07T01:14:27,696 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d2b1eb41dda894f4f7b826a2826a9c7b, ASSIGN 2024-12-07T01:14:27,696 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fa4e20ed49e59ee4cf63b6d84515872f, ASSIGN 2024-12-07T01:14:27,696 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 80bacdcc81a44a51d622e9d873892ea3, server=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:27,696 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1cf1ce3ee0822ff957b8ffa13f56df7d, server=ec1863dc21e5,40693,1733534061744 2024-12-07T01:14:27,696 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 37d131a6ac713be02dc73a29079210f5, server=ec1863dc21e5,40693,1733534061744 2024-12-07T01:14:27,696 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure d2b1eb41dda894f4f7b826a2826a9c7b, server=ec1863dc21e5,40693,1733534061744 2024-12-07T01:14:27,696 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure fa4e20ed49e59ee4cf63b6d84515872f, server=ec1863dc21e5,40693,1733534061744 2024-12-07T01:14:27,696 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6d8689bfa03ba7aed23b15e5ff5d23ac, server=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:27,696 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5d3306d206c48a1c7060e49d4fe00ad5, server=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:27,696 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 324bdaae20264b3880845c3082304e57, server=ec1863dc21e5,42915,1733534061580:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ec1863dc21e5,36079,1733534061701, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:27,696 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ec1863dc21e5%2C36079%2C1733534061701.meta.1733534063927.meta, worker=ec1863dc21e5,42915,1733534061580 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:27,697 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ec1863dc21e5%2C36079%2C1733534061701.meta.1733534063927.meta, worker=ec1863dc21e5,42915,1733534061580 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:27,697 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:27,697 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ec1863dc21e5,40693,1733534061744 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:27,697 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ec1863dc21e5%2C36079%2C1733534061701.1733534063448, worker=ec1863dc21e5,42915,1733534061580 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:27,697 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ec1863dc21e5%2C36079%2C1733534061701.1733534063448, worker=ec1863dc21e5,42915,1733534061580 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:27,697 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=37d131a6ac713be02dc73a29079210f5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:27,697 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=37d131a6ac713be02dc73a29079210f5, ASSIGN 2024-12-07T01:14:27,697 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=324bdaae20264b3880845c3082304e57, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:27,697 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=324bdaae20264b3880845c3082304e57, ASSIGN 2024-12-07T01:14:27,697 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1cf1ce3ee0822ff957b8ffa13f56df7d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:27,697 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1cf1ce3ee0822ff957b8ffa13f56df7d, ASSIGN 2024-12-07T01:14:27,697 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6d8689bfa03ba7aed23b15e5ff5d23ac, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:27,697 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6d8689bfa03ba7aed23b15e5ff5d23ac, ASSIGN 2024-12-07T01:14:27,697 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5d3306d206c48a1c7060e49d4fe00ad5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:27,697 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5d3306d206c48a1c7060e49d4fe00ad5, ASSIGN 2024-12-07T01:14:27,697 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=80bacdcc81a44a51d622e9d873892ea3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:27,697 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=80bacdcc81a44a51d622e9d873892ea3, ASSIGN 2024-12-07T01:14:27,697 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d2b1eb41dda894f4f7b826a2826a9c7b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:27,697 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d2b1eb41dda894f4f7b826a2826a9c7b, ASSIGN 2024-12-07T01:14:27,697 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fa4e20ed49e59ee4cf63b6d84515872f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:27,697 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fa4e20ed49e59ee4cf63b6d84515872f, ASSIGN 2024-12-07T01:14:27,697 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 80bacdcc81a44a51d622e9d873892ea3, server=ec1863dc21e5,42915,1733534061580 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:27,697 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1cf1ce3ee0822ff957b8ffa13f56df7d, server=ec1863dc21e5,40693,1733534061744 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:27,697 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 37d131a6ac713be02dc73a29079210f5, server=ec1863dc21e5,40693,1733534061744 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:27,697 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure d2b1eb41dda894f4f7b826a2826a9c7b, server=ec1863dc21e5,40693,1733534061744 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:27,697 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure fa4e20ed49e59ee4cf63b6d84515872f, server=ec1863dc21e5,40693,1733534061744 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:27,697 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6d8689bfa03ba7aed23b15e5ff5d23ac, server=ec1863dc21e5,42915,1733534061580 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:27,697 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5d3306d206c48a1c7060e49d4fe00ad5, server=ec1863dc21e5,42915,1733534061580 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:27,698 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 9 msec 2024-12-07T01:14:27,699 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=61 2024-12-07T01:14:27,699 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=ec1863dc21e5,40693,1733534061744, table=hbase:meta, region=1588230740 2024-12-07T01:14:27,699 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ec1863dc21e5,40693,1733534061744, state=OPEN 2024-12-07T01:14:27,711 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46141-0x101ad5a899b0000, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T01:14:27,711 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42915-0x101ad5a899b0001, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T01:14:27,711 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40693-0x101ad5a899b0003, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T01:14:27,712 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T01:14:27,712 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T01:14:27,712 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T01:14:27,712 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=37d131a6ac713be02dc73a29079210f5, ASSIGN to state=OFFLINE, location=null, table=test, region=37d131a6ac713be02dc73a29079210f5 to restore RIT 2024-12-07T01:14:27,712 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=324bdaae20264b3880845c3082304e57, ASSIGN to state=OFFLINE, location=null, table=test, region=324bdaae20264b3880845c3082304e57 to restore RIT 2024-12-07T01:14:27,712 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1cf1ce3ee0822ff957b8ffa13f56df7d, ASSIGN to state=OFFLINE, location=null, table=test, region=1cf1ce3ee0822ff957b8ffa13f56df7d to restore RIT 2024-12-07T01:14:27,712 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6d8689bfa03ba7aed23b15e5ff5d23ac, ASSIGN to state=OFFLINE, location=null, table=test, region=6d8689bfa03ba7aed23b15e5ff5d23ac to restore RIT 2024-12-07T01:14:27,712 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5d3306d206c48a1c7060e49d4fe00ad5, ASSIGN to state=OFFLINE, location=null, table=test, region=5d3306d206c48a1c7060e49d4fe00ad5 to restore RIT 2024-12-07T01:14:27,712 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=80bacdcc81a44a51d622e9d873892ea3, ASSIGN to state=OFFLINE, location=null, table=test, region=80bacdcc81a44a51d622e9d873892ea3 to restore RIT 2024-12-07T01:14:27,712 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d2b1eb41dda894f4f7b826a2826a9c7b, ASSIGN to state=OFFLINE, location=null, table=test, region=d2b1eb41dda894f4f7b826a2826a9c7b to restore RIT 2024-12-07T01:14:27,713 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fa4e20ed49e59ee4cf63b6d84515872f, ASSIGN to state=OFFLINE, location=null, table=test, region=fa4e20ed49e59ee4cf63b6d84515872f to restore RIT 2024-12-07T01:14:27,713 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733534097713 2024-12-07T01:14:27,713 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-07T01:14:27,713 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 324bdaae20264b3880845c3082304e57, server=ec1863dc21e5,42915,1733534061580:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ec1863dc21e5,36079,1733534061701, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-07T01:14:27,714 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-07T01:14:27,715 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-07T01:14:27,715 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=77, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6d8689bfa03ba7aed23b15e5ff5d23ac, server=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:27,715 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-07T01:14:27,715 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=77, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6d8689bfa03ba7aed23b15e5ff5d23ac, server=ec1863dc21e5,42915,1733534061580 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-07T01:14:27,720 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=976bbe8d7afc3b1e02af11d1b515e82c, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,720 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0980fd6969c12f09b843d772c626dfa5, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,720 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d8689bfa03ba7aed23b15e5ff5d23ac, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,720 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=af843c9df2505d2eeee3c18062e9a2a6, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,720 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c0a570532487ea0ebbc51cbec9ff7e37, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,720 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=37d131a6ac713be02dc73a29079210f5, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,721 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c0f43810347933724c9e6319b11ae5cc, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,721 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9c426833151f173ef0d81d9c0ae11767, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,721 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5d3306d206c48a1c7060e49d4fe00ad5, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,721 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aa3dca7b6c799eb77565fb0e20ef1c15, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,721 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=75c927364905f2c6bffff16328d8aaf5, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,721 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d2b1eb41dda894f4f7b826a2826a9c7b, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,721 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=327d2bdcc20803fcfd1e61553f30068b, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,722 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3fe1b6a546d269c366ae3f13263e1a2a, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,722 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=324bdaae20264b3880845c3082304e57, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,722 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c66f402dd9e22edaa5996a3b7446808c, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,722 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=856730db89c49e65ccb40b5967ccf99d, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,722 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fa4e20ed49e59ee4cf63b6d84515872f, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,722 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1da36e78b320d37341113b4185c91b26, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,722 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3f148a9f8c5bea63f4744afb36553ef7, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,723 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=80bacdcc81a44a51d622e9d873892ea3, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,723 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=76cd5c1764feda4c992319b218572cef, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,723 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8cd2040dc039667d68bc1a24f5f2322f, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,723 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1cf1ce3ee0822ff957b8ffa13f56df7d, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,723 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b59f40eb016096571b13ae445f262449, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,723 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=73ec7ce1c93542e31d99b7b4474a9750, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,723 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-07T01:14:27,723 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733534127723 2024-12-07T01:14:27,723 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733534187723 2024-12-07T01:14:27,723 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 10 msec 2024-12-07T01:14:27,723 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-07T01:14:27,723 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-07T01:14:27,723 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-07T01:14:27,724 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-07T01:14:27,724 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-07T01:14:27,724 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-07T01:14:27,724 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-07T01:14:27,724 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-07T01:14:27,725 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-07T01:14:27,727 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-07T01:14:27,727 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T01:14:27,727 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=64, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-07T01:14:27,728 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T01:14:27,728 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T01:14:27,728 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-07T01:14:27,728 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T01:14:27,728 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-07T01:14:27,728 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T01:14:27,728 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T01:14:27,728 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-07T01:14:27,728 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T01:14:27,728 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-07T01:14:27,728 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T01:14:27,728 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-07T01:14:27,728 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-07T01:14:27,728 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-07T01:14:27,728 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T01:14:27,728 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-07T01:14:27,728 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-07T01:14:27,728 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-07T01:14:27,728 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-07T01:14:27,729 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-07T01:14:27,730 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 324bdaae20264b3880845c3082304e57, server=ec1863dc21e5,42915,1733534061580:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ec1863dc21e5,36079,1733534061701, splitWal=true, meta=true 2024-12-07T01:14:27,731 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ec1863dc21e5%2C36079%2C1733534061701.meta.1733534063927.meta, worker=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:27,731 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ec1863dc21e5%2C36079%2C1733534061701.meta.1733534063927.meta, worker=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:27,731 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-07T01:14:27,731 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ec1863dc21e5,40693,1733534061744 2024-12-07T01:14:27,731 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ec1863dc21e5%2C36079%2C1733534061701.1733534063448, worker=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:27,731 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ec1863dc21e5%2C36079%2C1733534061701.1733534063448, worker=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:27,731 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=37d131a6ac713be02dc73a29079210f5, ASSIGN 2024-12-07T01:14:27,731 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=324bdaae20264b3880845c3082304e57, ASSIGN 2024-12-07T01:14:27,731 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1cf1ce3ee0822ff957b8ffa13f56df7d, ASSIGN 2024-12-07T01:14:27,731 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6d8689bfa03ba7aed23b15e5ff5d23ac, ASSIGN 2024-12-07T01:14:27,731 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5d3306d206c48a1c7060e49d4fe00ad5, ASSIGN 2024-12-07T01:14:27,731 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=80bacdcc81a44a51d622e9d873892ea3, ASSIGN 2024-12-07T01:14:27,731 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d2b1eb41dda894f4f7b826a2826a9c7b, ASSIGN 2024-12-07T01:14:27,731 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fa4e20ed49e59ee4cf63b6d84515872f, ASSIGN 2024-12-07T01:14:27,731 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 80bacdcc81a44a51d622e9d873892ea3, server=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:27,731 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1cf1ce3ee0822ff957b8ffa13f56df7d, server=ec1863dc21e5,40693,1733534061744 2024-12-07T01:14:27,731 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 37d131a6ac713be02dc73a29079210f5, server=ec1863dc21e5,40693,1733534061744 2024-12-07T01:14:27,731 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure d2b1eb41dda894f4f7b826a2826a9c7b, server=ec1863dc21e5,40693,1733534061744 2024-12-07T01:14:27,731 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure fa4e20ed49e59ee4cf63b6d84515872f, server=ec1863dc21e5,40693,1733534061744 2024-12-07T01:14:27,731 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6d8689bfa03ba7aed23b15e5ff5d23ac, server=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:27,731 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 324bdaae20264b3880845c3082304e57, server=ec1863dc21e5,42915,1733534061580:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ec1863dc21e5,36079,1733534061701, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:27,731 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ec1863dc21e5%2C36079%2C1733534061701.meta.1733534063927.meta, worker=ec1863dc21e5,42915,1733534061580 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:27,731 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ec1863dc21e5%2C36079%2C1733534061701.meta.1733534063927.meta, worker=ec1863dc21e5,42915,1733534061580 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:27,731 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:27,732 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ec1863dc21e5,40693,1733534061744 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:27,732 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ec1863dc21e5%2C36079%2C1733534061701.1733534063448, worker=ec1863dc21e5,42915,1733534061580 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:27,732 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ec1863dc21e5%2C36079%2C1733534061701.1733534063448, worker=ec1863dc21e5,42915,1733534061580 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:27,732 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=37d131a6ac713be02dc73a29079210f5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:27,732 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=37d131a6ac713be02dc73a29079210f5, ASSIGN 2024-12-07T01:14:27,732 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=324bdaae20264b3880845c3082304e57, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:27,732 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=324bdaae20264b3880845c3082304e57, ASSIGN 2024-12-07T01:14:27,732 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1cf1ce3ee0822ff957b8ffa13f56df7d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:27,732 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1cf1ce3ee0822ff957b8ffa13f56df7d, ASSIGN 2024-12-07T01:14:27,732 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6d8689bfa03ba7aed23b15e5ff5d23ac, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:27,732 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6d8689bfa03ba7aed23b15e5ff5d23ac, ASSIGN 2024-12-07T01:14:27,732 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5d3306d206c48a1c7060e49d4fe00ad5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:27,732 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5d3306d206c48a1c7060e49d4fe00ad5, ASSIGN 2024-12-07T01:14:27,732 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=80bacdcc81a44a51d622e9d873892ea3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:27,732 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=80bacdcc81a44a51d622e9d873892ea3, ASSIGN 2024-12-07T01:14:27,732 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d2b1eb41dda894f4f7b826a2826a9c7b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:27,732 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d2b1eb41dda894f4f7b826a2826a9c7b, ASSIGN 2024-12-07T01:14:27,732 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fa4e20ed49e59ee4cf63b6d84515872f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:27,732 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fa4e20ed49e59ee4cf63b6d84515872f, ASSIGN 2024-12-07T01:14:27,732 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 80bacdcc81a44a51d622e9d873892ea3, server=ec1863dc21e5,42915,1733534061580 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:27,732 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1cf1ce3ee0822ff957b8ffa13f56df7d, server=ec1863dc21e5,40693,1733534061744 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:27,732 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 37d131a6ac713be02dc73a29079210f5, server=ec1863dc21e5,40693,1733534061744 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:27,732 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure d2b1eb41dda894f4f7b826a2826a9c7b, server=ec1863dc21e5,40693,1733534061744 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:27,732 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure fa4e20ed49e59ee4cf63b6d84515872f, server=ec1863dc21e5,40693,1733534061744 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:27,732 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6d8689bfa03ba7aed23b15e5ff5d23ac, server=ec1863dc21e5,42915,1733534061580 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:27,732 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-12-07T01:14:27,733 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=61 2024-12-07T01:14:27,733 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=ec1863dc21e5,40693,1733534061744, table=hbase:meta, region=1588230740 2024-12-07T01:14:27,734 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ec1863dc21e5,40693,1733534061744, state=OPEN 2024-12-07T01:14:27,743 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46141-0x101ad5a899b0000, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T01:14:27,743 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42915-0x101ad5a899b0001, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T01:14:27,743 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40693-0x101ad5a899b0003, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T01:14:27,743 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T01:14:27,743 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T01:14:27,743 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T01:14:27,744 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=37d131a6ac713be02dc73a29079210f5, ASSIGN to state=OFFLINE, location=null, table=test, region=37d131a6ac713be02dc73a29079210f5 to restore RIT 2024-12-07T01:14:27,744 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=324bdaae20264b3880845c3082304e57, ASSIGN to state=OFFLINE, location=null, table=test, region=324bdaae20264b3880845c3082304e57 to restore RIT 2024-12-07T01:14:27,744 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1cf1ce3ee0822ff957b8ffa13f56df7d, ASSIGN to state=OFFLINE, location=null, table=test, region=1cf1ce3ee0822ff957b8ffa13f56df7d to restore RIT 2024-12-07T01:14:27,744 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6d8689bfa03ba7aed23b15e5ff5d23ac, ASSIGN to state=OFFLINE, location=null, table=test, region=6d8689bfa03ba7aed23b15e5ff5d23ac to restore RIT 2024-12-07T01:14:27,744 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5d3306d206c48a1c7060e49d4fe00ad5, ASSIGN to state=OFFLINE, location=null, table=test, region=5d3306d206c48a1c7060e49d4fe00ad5 to restore RIT 2024-12-07T01:14:27,744 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=80bacdcc81a44a51d622e9d873892ea3, ASSIGN to state=OFFLINE, location=null, table=test, region=80bacdcc81a44a51d622e9d873892ea3 to restore RIT 2024-12-07T01:14:27,744 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d2b1eb41dda894f4f7b826a2826a9c7b, ASSIGN to state=OFFLINE, location=null, table=test, region=d2b1eb41dda894f4f7b826a2826a9c7b to restore RIT 2024-12-07T01:14:27,744 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fa4e20ed49e59ee4cf63b6d84515872f, ASSIGN to state=OFFLINE, location=null, table=test, region=fa4e20ed49e59ee4cf63b6d84515872f to restore RIT 2024-12-07T01:14:27,745 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733534097744 2024-12-07T01:14:27,745 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-07T01:14:27,745 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 324bdaae20264b3880845c3082304e57, server=ec1863dc21e5,42915,1733534061580:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ec1863dc21e5,36079,1733534061701, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-07T01:14:27,746 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-07T01:14:27,747 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-07T01:14:27,747 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=76, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure fa4e20ed49e59ee4cf63b6d84515872f, server=ec1863dc21e5,40693,1733534061744 2024-12-07T01:14:27,747 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-07T01:14:27,747 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=76, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure fa4e20ed49e59ee4cf63b6d84515872f, server=ec1863dc21e5,40693,1733534061744 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-07T01:14:27,751 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=976bbe8d7afc3b1e02af11d1b515e82c, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,751 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0980fd6969c12f09b843d772c626dfa5, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,751 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d8689bfa03ba7aed23b15e5ff5d23ac, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,751 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=af843c9df2505d2eeee3c18062e9a2a6, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,751 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c0a570532487ea0ebbc51cbec9ff7e37, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,751 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=37d131a6ac713be02dc73a29079210f5, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,751 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c0f43810347933724c9e6319b11ae5cc, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,752 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9c426833151f173ef0d81d9c0ae11767, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,752 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5d3306d206c48a1c7060e49d4fe00ad5, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,752 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aa3dca7b6c799eb77565fb0e20ef1c15, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,752 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=75c927364905f2c6bffff16328d8aaf5, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,752 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d2b1eb41dda894f4f7b826a2826a9c7b, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,752 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=327d2bdcc20803fcfd1e61553f30068b, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,752 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3fe1b6a546d269c366ae3f13263e1a2a, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,752 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=324bdaae20264b3880845c3082304e57, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,753 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c66f402dd9e22edaa5996a3b7446808c, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,753 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=856730db89c49e65ccb40b5967ccf99d, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,753 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fa4e20ed49e59ee4cf63b6d84515872f, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,753 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1da36e78b320d37341113b4185c91b26, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,753 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3f148a9f8c5bea63f4744afb36553ef7, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,753 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=80bacdcc81a44a51d622e9d873892ea3, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,753 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=76cd5c1764feda4c992319b218572cef, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,753 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8cd2040dc039667d68bc1a24f5f2322f, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,754 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1cf1ce3ee0822ff957b8ffa13f56df7d, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,754 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b59f40eb016096571b13ae445f262449, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,754 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=73ec7ce1c93542e31d99b7b4474a9750, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,754 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-07T01:14:27,754 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733534127754 2024-12-07T01:14:27,754 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733534187754 2024-12-07T01:14:27,754 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-12-07T01:14:27,754 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-07T01:14:27,754 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-07T01:14:27,754 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-07T01:14:27,754 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-07T01:14:27,754 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-07T01:14:27,754 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-07T01:14:27,755 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-07T01:14:27,755 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-07T01:14:27,755 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-07T01:14:27,756 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46141 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T01:14:27,757 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46141 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T01:14:27,758 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-07T01:14:27,758 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T01:14:27,758 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=64, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-07T01:14:27,758 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T01:14:27,758 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T01:14:27,758 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-07T01:14:27,758 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T01:14:27,758 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T01:14:27,758 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T01:14:27,758 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-07T01:14:27,758 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T01:14:27,758 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-07T01:14:27,758 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T01:14:27,759 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-07T01:14:27,759 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-07T01:14:27,759 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-07T01:14:27,759 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T01:14:27,759 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-07T01:14:27,759 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-07T01:14:27,759 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-07T01:14:27,759 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-07T01:14:27,759 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-07T01:14:27,761 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 324bdaae20264b3880845c3082304e57, server=ec1863dc21e5,42915,1733534061580:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ec1863dc21e5,36079,1733534061701, splitWal=true, meta=true 2024-12-07T01:14:27,761 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ec1863dc21e5%2C36079%2C1733534061701.meta.1733534063927.meta, worker=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:27,761 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ec1863dc21e5%2C36079%2C1733534061701.meta.1733534063927.meta, worker=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:27,761 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-07T01:14:27,761 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ec1863dc21e5,40693,1733534061744 2024-12-07T01:14:27,761 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ec1863dc21e5%2C36079%2C1733534061701.1733534063448, worker=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:27,761 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ec1863dc21e5%2C36079%2C1733534061701.1733534063448, worker=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:27,761 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=37d131a6ac713be02dc73a29079210f5, ASSIGN 2024-12-07T01:14:27,761 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=324bdaae20264b3880845c3082304e57, ASSIGN 2024-12-07T01:14:27,761 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1cf1ce3ee0822ff957b8ffa13f56df7d, ASSIGN 2024-12-07T01:14:27,761 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6d8689bfa03ba7aed23b15e5ff5d23ac, ASSIGN 2024-12-07T01:14:27,761 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5d3306d206c48a1c7060e49d4fe00ad5, ASSIGN 2024-12-07T01:14:27,761 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=80bacdcc81a44a51d622e9d873892ea3, ASSIGN 2024-12-07T01:14:27,761 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d2b1eb41dda894f4f7b826a2826a9c7b, ASSIGN 2024-12-07T01:14:27,761 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fa4e20ed49e59ee4cf63b6d84515872f, ASSIGN 2024-12-07T01:14:27,762 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 80bacdcc81a44a51d622e9d873892ea3, server=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:27,762 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1cf1ce3ee0822ff957b8ffa13f56df7d, server=ec1863dc21e5,40693,1733534061744 2024-12-07T01:14:27,762 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 37d131a6ac713be02dc73a29079210f5, server=ec1863dc21e5,40693,1733534061744 2024-12-07T01:14:27,762 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure d2b1eb41dda894f4f7b826a2826a9c7b, server=ec1863dc21e5,40693,1733534061744 2024-12-07T01:14:27,762 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure fa4e20ed49e59ee4cf63b6d84515872f, server=ec1863dc21e5,40693,1733534061744 2024-12-07T01:14:27,762 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 324bdaae20264b3880845c3082304e57, server=ec1863dc21e5,42915,1733534061580:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ec1863dc21e5,36079,1733534061701, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:27,762 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ec1863dc21e5%2C36079%2C1733534061701.meta.1733534063927.meta, worker=ec1863dc21e5,42915,1733534061580 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:27,762 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ec1863dc21e5%2C36079%2C1733534061701.meta.1733534063927.meta, worker=ec1863dc21e5,42915,1733534061580 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:27,762 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:27,762 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ec1863dc21e5,40693,1733534061744 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:27,762 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ec1863dc21e5%2C36079%2C1733534061701.1733534063448, worker=ec1863dc21e5,42915,1733534061580 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:27,762 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ec1863dc21e5%2C36079%2C1733534061701.1733534063448, worker=ec1863dc21e5,42915,1733534061580 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:27,762 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=37d131a6ac713be02dc73a29079210f5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:27,762 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=37d131a6ac713be02dc73a29079210f5, ASSIGN 2024-12-07T01:14:27,762 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=324bdaae20264b3880845c3082304e57, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:27,762 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=324bdaae20264b3880845c3082304e57, ASSIGN 2024-12-07T01:14:27,762 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1cf1ce3ee0822ff957b8ffa13f56df7d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:27,762 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1cf1ce3ee0822ff957b8ffa13f56df7d, ASSIGN 2024-12-07T01:14:27,762 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6d8689bfa03ba7aed23b15e5ff5d23ac, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:27,762 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6d8689bfa03ba7aed23b15e5ff5d23ac, ASSIGN 2024-12-07T01:14:27,762 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5d3306d206c48a1c7060e49d4fe00ad5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:27,762 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5d3306d206c48a1c7060e49d4fe00ad5, ASSIGN 2024-12-07T01:14:27,762 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=80bacdcc81a44a51d622e9d873892ea3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:27,762 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=80bacdcc81a44a51d622e9d873892ea3, ASSIGN 2024-12-07T01:14:27,762 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d2b1eb41dda894f4f7b826a2826a9c7b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:27,762 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d2b1eb41dda894f4f7b826a2826a9c7b, ASSIGN 2024-12-07T01:14:27,762 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fa4e20ed49e59ee4cf63b6d84515872f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:27,763 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fa4e20ed49e59ee4cf63b6d84515872f, ASSIGN 2024-12-07T01:14:27,763 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 80bacdcc81a44a51d622e9d873892ea3, server=ec1863dc21e5,42915,1733534061580 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:27,763 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1cf1ce3ee0822ff957b8ffa13f56df7d, server=ec1863dc21e5,40693,1733534061744 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:27,763 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 37d131a6ac713be02dc73a29079210f5, server=ec1863dc21e5,40693,1733534061744 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:27,763 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure d2b1eb41dda894f4f7b826a2826a9c7b, server=ec1863dc21e5,40693,1733534061744 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:27,763 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure fa4e20ed49e59ee4cf63b6d84515872f, server=ec1863dc21e5,40693,1733534061744 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:27,763 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-12-07T01:14:27,764 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=61 2024-12-07T01:14:27,764 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=ec1863dc21e5,40693,1733534061744, table=hbase:meta, region=1588230740 2024-12-07T01:14:27,764 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ec1863dc21e5,40693,1733534061744, state=OPEN 2024-12-07T01:14:27,775 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40693-0x101ad5a899b0003, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T01:14:27,775 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46141-0x101ad5a899b0000, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T01:14:27,775 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42915-0x101ad5a899b0001, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T01:14:27,775 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T01:14:27,775 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T01:14:27,775 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T01:14:27,775 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=37d131a6ac713be02dc73a29079210f5, ASSIGN to state=OFFLINE, location=null, table=test, region=37d131a6ac713be02dc73a29079210f5 to restore RIT 2024-12-07T01:14:27,776 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=324bdaae20264b3880845c3082304e57, ASSIGN to state=OFFLINE, location=null, table=test, region=324bdaae20264b3880845c3082304e57 to restore RIT 2024-12-07T01:14:27,776 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1cf1ce3ee0822ff957b8ffa13f56df7d, ASSIGN to state=OFFLINE, location=null, table=test, region=1cf1ce3ee0822ff957b8ffa13f56df7d to restore RIT 2024-12-07T01:14:27,776 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6d8689bfa03ba7aed23b15e5ff5d23ac, ASSIGN to state=OFFLINE, location=null, table=test, region=6d8689bfa03ba7aed23b15e5ff5d23ac to restore RIT 2024-12-07T01:14:27,776 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5d3306d206c48a1c7060e49d4fe00ad5, ASSIGN to state=OFFLINE, location=null, table=test, region=5d3306d206c48a1c7060e49d4fe00ad5 to restore RIT 2024-12-07T01:14:27,776 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=80bacdcc81a44a51d622e9d873892ea3, ASSIGN to state=OFFLINE, location=null, table=test, region=80bacdcc81a44a51d622e9d873892ea3 to restore RIT 2024-12-07T01:14:27,776 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d2b1eb41dda894f4f7b826a2826a9c7b, ASSIGN to state=OFFLINE, location=null, table=test, region=d2b1eb41dda894f4f7b826a2826a9c7b to restore RIT 2024-12-07T01:14:27,776 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fa4e20ed49e59ee4cf63b6d84515872f, ASSIGN to state=OFFLINE, location=null, table=test, region=fa4e20ed49e59ee4cf63b6d84515872f to restore RIT 2024-12-07T01:14:27,776 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733534097776 2024-12-07T01:14:27,776 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-07T01:14:27,776 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 324bdaae20264b3880845c3082304e57, server=ec1863dc21e5,42915,1733534061580:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ec1863dc21e5,36079,1733534061701, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-07T01:14:27,778 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-07T01:14:27,778 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-07T01:14:27,779 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=75, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure d2b1eb41dda894f4f7b826a2826a9c7b, server=ec1863dc21e5,40693,1733534061744 2024-12-07T01:14:27,779 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-07T01:14:27,779 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=75, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure d2b1eb41dda894f4f7b826a2826a9c7b, server=ec1863dc21e5,40693,1733534061744 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-07T01:14:27,782 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=976bbe8d7afc3b1e02af11d1b515e82c, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,782 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0980fd6969c12f09b843d772c626dfa5, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,782 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d8689bfa03ba7aed23b15e5ff5d23ac, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,782 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=af843c9df2505d2eeee3c18062e9a2a6, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,782 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c0a570532487ea0ebbc51cbec9ff7e37, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,783 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=37d131a6ac713be02dc73a29079210f5, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,783 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c0f43810347933724c9e6319b11ae5cc, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,783 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9c426833151f173ef0d81d9c0ae11767, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,783 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5d3306d206c48a1c7060e49d4fe00ad5, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,783 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aa3dca7b6c799eb77565fb0e20ef1c15, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,783 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=75c927364905f2c6bffff16328d8aaf5, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,783 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d2b1eb41dda894f4f7b826a2826a9c7b, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,784 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=327d2bdcc20803fcfd1e61553f30068b, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,784 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3fe1b6a546d269c366ae3f13263e1a2a, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,784 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=324bdaae20264b3880845c3082304e57, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,784 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c66f402dd9e22edaa5996a3b7446808c, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,784 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=856730db89c49e65ccb40b5967ccf99d, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,784 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fa4e20ed49e59ee4cf63b6d84515872f, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,784 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1da36e78b320d37341113b4185c91b26, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,784 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3f148a9f8c5bea63f4744afb36553ef7, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,785 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=80bacdcc81a44a51d622e9d873892ea3, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,785 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=76cd5c1764feda4c992319b218572cef, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,785 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8cd2040dc039667d68bc1a24f5f2322f, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,785 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1cf1ce3ee0822ff957b8ffa13f56df7d, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,785 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b59f40eb016096571b13ae445f262449, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,785 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=73ec7ce1c93542e31d99b7b4474a9750, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,785 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-07T01:14:27,785 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733534127785 2024-12-07T01:14:27,785 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733534187785 2024-12-07T01:14:27,785 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-12-07T01:14:27,785 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-07T01:14:27,785 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-07T01:14:27,785 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-07T01:14:27,785 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-07T01:14:27,786 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-07T01:14:27,786 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-07T01:14:27,786 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-07T01:14:27,786 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-07T01:14:27,787 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-07T01:14:27,789 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-07T01:14:27,789 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T01:14:27,789 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=64, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-07T01:14:27,789 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T01:14:27,789 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T01:14:27,789 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-07T01:14:27,789 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T01:14:27,789 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T01:14:27,789 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T01:14:27,789 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-07T01:14:27,789 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T01:14:27,789 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-07T01:14:27,789 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T01:14:27,789 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-07T01:14:27,789 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-07T01:14:27,789 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T01:14:27,789 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-07T01:14:27,789 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-07T01:14:27,789 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-07T01:14:27,790 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-07T01:14:27,790 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-07T01:14:27,791 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 324bdaae20264b3880845c3082304e57, server=ec1863dc21e5,42915,1733534061580:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ec1863dc21e5,36079,1733534061701, splitWal=true, meta=true 2024-12-07T01:14:27,792 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ec1863dc21e5%2C36079%2C1733534061701.meta.1733534063927.meta, worker=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:27,792 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ec1863dc21e5%2C36079%2C1733534061701.meta.1733534063927.meta, worker=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:27,792 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-07T01:14:27,792 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ec1863dc21e5,40693,1733534061744 2024-12-07T01:14:27,792 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ec1863dc21e5%2C36079%2C1733534061701.1733534063448, worker=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:27,792 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ec1863dc21e5%2C36079%2C1733534061701.1733534063448, worker=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:27,792 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=37d131a6ac713be02dc73a29079210f5, ASSIGN 2024-12-07T01:14:27,792 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=324bdaae20264b3880845c3082304e57, ASSIGN 2024-12-07T01:14:27,792 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1cf1ce3ee0822ff957b8ffa13f56df7d, ASSIGN 2024-12-07T01:14:27,792 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6d8689bfa03ba7aed23b15e5ff5d23ac, ASSIGN 2024-12-07T01:14:27,792 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5d3306d206c48a1c7060e49d4fe00ad5, ASSIGN 2024-12-07T01:14:27,792 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=80bacdcc81a44a51d622e9d873892ea3, ASSIGN 2024-12-07T01:14:27,792 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d2b1eb41dda894f4f7b826a2826a9c7b, ASSIGN 2024-12-07T01:14:27,792 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fa4e20ed49e59ee4cf63b6d84515872f, ASSIGN 2024-12-07T01:14:27,792 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 80bacdcc81a44a51d622e9d873892ea3, server=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:27,792 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1cf1ce3ee0822ff957b8ffa13f56df7d, server=ec1863dc21e5,40693,1733534061744 2024-12-07T01:14:27,792 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 37d131a6ac713be02dc73a29079210f5, server=ec1863dc21e5,40693,1733534061744 2024-12-07T01:14:27,792 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure d2b1eb41dda894f4f7b826a2826a9c7b, server=ec1863dc21e5,40693,1733534061744 2024-12-07T01:14:27,792 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 324bdaae20264b3880845c3082304e57, server=ec1863dc21e5,42915,1733534061580:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ec1863dc21e5,36079,1733534061701, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:27,792 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ec1863dc21e5%2C36079%2C1733534061701.meta.1733534063927.meta, worker=ec1863dc21e5,42915,1733534061580 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:27,792 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ec1863dc21e5%2C36079%2C1733534061701.meta.1733534063927.meta, worker=ec1863dc21e5,42915,1733534061580 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:27,792 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:27,792 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ec1863dc21e5,40693,1733534061744 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:27,792 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ec1863dc21e5%2C36079%2C1733534061701.1733534063448, worker=ec1863dc21e5,42915,1733534061580 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:27,792 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ec1863dc21e5%2C36079%2C1733534061701.1733534063448, worker=ec1863dc21e5,42915,1733534061580 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:27,792 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=37d131a6ac713be02dc73a29079210f5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:27,792 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=37d131a6ac713be02dc73a29079210f5, ASSIGN 2024-12-07T01:14:27,793 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=324bdaae20264b3880845c3082304e57, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:27,793 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=324bdaae20264b3880845c3082304e57, ASSIGN 2024-12-07T01:14:27,793 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1cf1ce3ee0822ff957b8ffa13f56df7d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:27,793 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1cf1ce3ee0822ff957b8ffa13f56df7d, ASSIGN 2024-12-07T01:14:27,793 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6d8689bfa03ba7aed23b15e5ff5d23ac, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:27,793 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6d8689bfa03ba7aed23b15e5ff5d23ac, ASSIGN 2024-12-07T01:14:27,793 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5d3306d206c48a1c7060e49d4fe00ad5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:27,793 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5d3306d206c48a1c7060e49d4fe00ad5, ASSIGN 2024-12-07T01:14:27,793 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=80bacdcc81a44a51d622e9d873892ea3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:27,793 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=80bacdcc81a44a51d622e9d873892ea3, ASSIGN 2024-12-07T01:14:27,793 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d2b1eb41dda894f4f7b826a2826a9c7b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:27,793 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d2b1eb41dda894f4f7b826a2826a9c7b, ASSIGN 2024-12-07T01:14:27,793 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fa4e20ed49e59ee4cf63b6d84515872f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:27,793 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fa4e20ed49e59ee4cf63b6d84515872f, ASSIGN 2024-12-07T01:14:27,793 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 80bacdcc81a44a51d622e9d873892ea3, server=ec1863dc21e5,42915,1733534061580 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:27,793 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1cf1ce3ee0822ff957b8ffa13f56df7d, server=ec1863dc21e5,40693,1733534061744 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:27,793 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 37d131a6ac713be02dc73a29079210f5, server=ec1863dc21e5,40693,1733534061744 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:27,793 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure d2b1eb41dda894f4f7b826a2826a9c7b, server=ec1863dc21e5,40693,1733534061744 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:27,793 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-12-07T01:14:27,794 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=61 2024-12-07T01:14:27,794 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=ec1863dc21e5,40693,1733534061744, table=hbase:meta, region=1588230740 2024-12-07T01:14:27,794 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ec1863dc21e5,40693,1733534061744, state=OPEN 2024-12-07T01:14:27,806 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40693-0x101ad5a899b0003, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T01:14:27,806 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42915-0x101ad5a899b0001, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T01:14:27,806 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46141-0x101ad5a899b0000, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T01:14:27,806 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T01:14:27,806 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T01:14:27,806 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T01:14:27,807 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=37d131a6ac713be02dc73a29079210f5, ASSIGN to state=OFFLINE, location=null, table=test, region=37d131a6ac713be02dc73a29079210f5 to restore RIT 2024-12-07T01:14:27,807 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=324bdaae20264b3880845c3082304e57, ASSIGN to state=OFFLINE, location=null, table=test, region=324bdaae20264b3880845c3082304e57 to restore RIT 2024-12-07T01:14:27,807 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1cf1ce3ee0822ff957b8ffa13f56df7d, ASSIGN to state=OFFLINE, location=null, table=test, region=1cf1ce3ee0822ff957b8ffa13f56df7d to restore RIT 2024-12-07T01:14:27,807 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6d8689bfa03ba7aed23b15e5ff5d23ac, ASSIGN to state=OFFLINE, location=null, table=test, region=6d8689bfa03ba7aed23b15e5ff5d23ac to restore RIT 2024-12-07T01:14:27,807 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5d3306d206c48a1c7060e49d4fe00ad5, ASSIGN to state=OFFLINE, location=null, table=test, region=5d3306d206c48a1c7060e49d4fe00ad5 to restore RIT 2024-12-07T01:14:27,807 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=80bacdcc81a44a51d622e9d873892ea3, ASSIGN to state=OFFLINE, location=null, table=test, region=80bacdcc81a44a51d622e9d873892ea3 to restore RIT 2024-12-07T01:14:27,807 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d2b1eb41dda894f4f7b826a2826a9c7b, ASSIGN to state=OFFLINE, location=null, table=test, region=d2b1eb41dda894f4f7b826a2826a9c7b to restore RIT 2024-12-07T01:14:27,807 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fa4e20ed49e59ee4cf63b6d84515872f, ASSIGN to state=OFFLINE, location=null, table=test, region=fa4e20ed49e59ee4cf63b6d84515872f to restore RIT 2024-12-07T01:14:27,808 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733534097807 2024-12-07T01:14:27,808 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-07T01:14:27,808 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 324bdaae20264b3880845c3082304e57, server=ec1863dc21e5,42915,1733534061580:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ec1863dc21e5,36079,1733534061701, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-07T01:14:27,809 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-07T01:14:27,810 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-07T01:14:27,810 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=74, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 37d131a6ac713be02dc73a29079210f5, server=ec1863dc21e5,40693,1733534061744 2024-12-07T01:14:27,810 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-07T01:14:27,810 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=74, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 37d131a6ac713be02dc73a29079210f5, server=ec1863dc21e5,40693,1733534061744 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-07T01:14:27,814 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=976bbe8d7afc3b1e02af11d1b515e82c, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,814 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0980fd6969c12f09b843d772c626dfa5, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,814 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d8689bfa03ba7aed23b15e5ff5d23ac, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,815 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=af843c9df2505d2eeee3c18062e9a2a6, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,815 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c0a570532487ea0ebbc51cbec9ff7e37, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,815 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=37d131a6ac713be02dc73a29079210f5, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,815 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c0f43810347933724c9e6319b11ae5cc, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,815 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9c426833151f173ef0d81d9c0ae11767, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,815 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5d3306d206c48a1c7060e49d4fe00ad5, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,815 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aa3dca7b6c799eb77565fb0e20ef1c15, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,815 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=75c927364905f2c6bffff16328d8aaf5, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,816 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d2b1eb41dda894f4f7b826a2826a9c7b, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,816 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=327d2bdcc20803fcfd1e61553f30068b, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,816 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3fe1b6a546d269c366ae3f13263e1a2a, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,816 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=324bdaae20264b3880845c3082304e57, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,816 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c66f402dd9e22edaa5996a3b7446808c, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,816 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=856730db89c49e65ccb40b5967ccf99d, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,816 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fa4e20ed49e59ee4cf63b6d84515872f, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,816 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1da36e78b320d37341113b4185c91b26, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,816 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3f148a9f8c5bea63f4744afb36553ef7, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,817 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=80bacdcc81a44a51d622e9d873892ea3, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,817 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=76cd5c1764feda4c992319b218572cef, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,817 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8cd2040dc039667d68bc1a24f5f2322f, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,817 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1cf1ce3ee0822ff957b8ffa13f56df7d, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,817 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b59f40eb016096571b13ae445f262449, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,817 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=73ec7ce1c93542e31d99b7b4474a9750, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,817 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-07T01:14:27,817 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733534127817 2024-12-07T01:14:27,817 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733534187817 2024-12-07T01:14:27,817 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-12-07T01:14:27,817 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-07T01:14:27,817 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-07T01:14:27,817 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-07T01:14:27,817 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-07T01:14:27,818 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-07T01:14:27,818 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-07T01:14:27,818 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-07T01:14:27,818 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-07T01:14:27,819 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-07T01:14:27,820 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-07T01:14:27,820 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T01:14:27,820 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=64, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-07T01:14:27,820 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T01:14:27,821 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T01:14:27,821 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-07T01:14:27,821 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T01:14:27,821 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T01:14:27,821 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T01:14:27,821 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-07T01:14:27,821 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T01:14:27,821 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T01:14:27,821 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-07T01:14:27,821 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-07T01:14:27,821 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T01:14:27,821 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-07T01:14:27,821 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-07T01:14:27,821 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-07T01:14:27,821 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-07T01:14:27,822 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-07T01:14:27,823 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 324bdaae20264b3880845c3082304e57, server=ec1863dc21e5,42915,1733534061580:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ec1863dc21e5,36079,1733534061701, splitWal=true, meta=true 2024-12-07T01:14:27,823 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ec1863dc21e5%2C36079%2C1733534061701.meta.1733534063927.meta, worker=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:27,823 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ec1863dc21e5%2C36079%2C1733534061701.meta.1733534063927.meta, worker=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:27,823 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-07T01:14:27,823 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ec1863dc21e5,40693,1733534061744 2024-12-07T01:14:27,823 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ec1863dc21e5%2C36079%2C1733534061701.1733534063448, worker=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:27,823 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ec1863dc21e5%2C36079%2C1733534061701.1733534063448, worker=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:27,823 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=37d131a6ac713be02dc73a29079210f5, ASSIGN 2024-12-07T01:14:27,823 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=324bdaae20264b3880845c3082304e57, ASSIGN 2024-12-07T01:14:27,823 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1cf1ce3ee0822ff957b8ffa13f56df7d, ASSIGN 2024-12-07T01:14:27,823 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6d8689bfa03ba7aed23b15e5ff5d23ac, ASSIGN 2024-12-07T01:14:27,823 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5d3306d206c48a1c7060e49d4fe00ad5, ASSIGN 2024-12-07T01:14:27,823 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=80bacdcc81a44a51d622e9d873892ea3, ASSIGN 2024-12-07T01:14:27,823 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d2b1eb41dda894f4f7b826a2826a9c7b, ASSIGN 2024-12-07T01:14:27,823 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fa4e20ed49e59ee4cf63b6d84515872f, ASSIGN 2024-12-07T01:14:27,823 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 80bacdcc81a44a51d622e9d873892ea3, server=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:27,824 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1cf1ce3ee0822ff957b8ffa13f56df7d, server=ec1863dc21e5,40693,1733534061744 2024-12-07T01:14:27,824 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 37d131a6ac713be02dc73a29079210f5, server=ec1863dc21e5,40693,1733534061744 2024-12-07T01:14:27,824 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 324bdaae20264b3880845c3082304e57, server=ec1863dc21e5,42915,1733534061580:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ec1863dc21e5,36079,1733534061701, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:27,824 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ec1863dc21e5%2C36079%2C1733534061701.meta.1733534063927.meta, worker=ec1863dc21e5,42915,1733534061580 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:27,824 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ec1863dc21e5%2C36079%2C1733534061701.meta.1733534063927.meta, worker=ec1863dc21e5,42915,1733534061580 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:27,824 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:27,824 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ec1863dc21e5,40693,1733534061744 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:27,824 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ec1863dc21e5%2C36079%2C1733534061701.1733534063448, worker=ec1863dc21e5,42915,1733534061580 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:27,824 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ec1863dc21e5%2C36079%2C1733534061701.1733534063448, worker=ec1863dc21e5,42915,1733534061580 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:27,824 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=37d131a6ac713be02dc73a29079210f5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:27,824 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=37d131a6ac713be02dc73a29079210f5, ASSIGN 2024-12-07T01:14:27,824 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=324bdaae20264b3880845c3082304e57, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:27,824 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=324bdaae20264b3880845c3082304e57, ASSIGN 2024-12-07T01:14:27,824 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1cf1ce3ee0822ff957b8ffa13f56df7d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:27,824 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1cf1ce3ee0822ff957b8ffa13f56df7d, ASSIGN 2024-12-07T01:14:27,824 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6d8689bfa03ba7aed23b15e5ff5d23ac, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:27,824 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6d8689bfa03ba7aed23b15e5ff5d23ac, ASSIGN 2024-12-07T01:14:27,824 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5d3306d206c48a1c7060e49d4fe00ad5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:27,824 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5d3306d206c48a1c7060e49d4fe00ad5, ASSIGN 2024-12-07T01:14:27,824 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=80bacdcc81a44a51d622e9d873892ea3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:27,824 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=80bacdcc81a44a51d622e9d873892ea3, ASSIGN 2024-12-07T01:14:27,824 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d2b1eb41dda894f4f7b826a2826a9c7b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:27,824 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d2b1eb41dda894f4f7b826a2826a9c7b, ASSIGN 2024-12-07T01:14:27,824 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fa4e20ed49e59ee4cf63b6d84515872f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:27,824 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fa4e20ed49e59ee4cf63b6d84515872f, ASSIGN 2024-12-07T01:14:27,824 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 80bacdcc81a44a51d622e9d873892ea3, server=ec1863dc21e5,42915,1733534061580 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:27,825 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1cf1ce3ee0822ff957b8ffa13f56df7d, server=ec1863dc21e5,40693,1733534061744 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:27,825 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 37d131a6ac713be02dc73a29079210f5, server=ec1863dc21e5,40693,1733534061744 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:27,825 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-12-07T01:14:27,825 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=61 2024-12-07T01:14:27,825 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=ec1863dc21e5,40693,1733534061744, table=hbase:meta, region=1588230740 2024-12-07T01:14:27,826 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ec1863dc21e5,40693,1733534061744, state=OPEN 2024-12-07T01:14:27,838 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46141-0x101ad5a899b0000, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T01:14:27,838 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42915-0x101ad5a899b0001, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T01:14:27,838 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40693-0x101ad5a899b0003, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T01:14:27,838 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T01:14:27,838 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T01:14:27,838 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T01:14:27,838 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=37d131a6ac713be02dc73a29079210f5, ASSIGN to state=OFFLINE, location=null, table=test, region=37d131a6ac713be02dc73a29079210f5 to restore RIT 2024-12-07T01:14:27,839 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=324bdaae20264b3880845c3082304e57, ASSIGN to state=OFFLINE, location=null, table=test, region=324bdaae20264b3880845c3082304e57 to restore RIT 2024-12-07T01:14:27,839 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1cf1ce3ee0822ff957b8ffa13f56df7d, ASSIGN to state=OFFLINE, location=null, table=test, region=1cf1ce3ee0822ff957b8ffa13f56df7d to restore RIT 2024-12-07T01:14:27,839 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6d8689bfa03ba7aed23b15e5ff5d23ac, ASSIGN to state=OFFLINE, location=null, table=test, region=6d8689bfa03ba7aed23b15e5ff5d23ac to restore RIT 2024-12-07T01:14:27,839 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5d3306d206c48a1c7060e49d4fe00ad5, ASSIGN to state=OFFLINE, location=null, table=test, region=5d3306d206c48a1c7060e49d4fe00ad5 to restore RIT 2024-12-07T01:14:27,839 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=80bacdcc81a44a51d622e9d873892ea3, ASSIGN to state=OFFLINE, location=null, table=test, region=80bacdcc81a44a51d622e9d873892ea3 to restore RIT 2024-12-07T01:14:27,839 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d2b1eb41dda894f4f7b826a2826a9c7b, ASSIGN to state=OFFLINE, location=null, table=test, region=d2b1eb41dda894f4f7b826a2826a9c7b to restore RIT 2024-12-07T01:14:27,839 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fa4e20ed49e59ee4cf63b6d84515872f, ASSIGN to state=OFFLINE, location=null, table=test, region=fa4e20ed49e59ee4cf63b6d84515872f to restore RIT 2024-12-07T01:14:27,839 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733534097839 2024-12-07T01:14:27,839 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-07T01:14:27,839 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 324bdaae20264b3880845c3082304e57, server=ec1863dc21e5,42915,1733534061580:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ec1863dc21e5,36079,1733534061701, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-07T01:14:27,840 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-07T01:14:27,841 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-07T01:14:27,841 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=73, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1cf1ce3ee0822ff957b8ffa13f56df7d, server=ec1863dc21e5,40693,1733534061744 2024-12-07T01:14:27,841 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-07T01:14:27,841 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=73, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1cf1ce3ee0822ff957b8ffa13f56df7d, server=ec1863dc21e5,40693,1733534061744 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-07T01:14:27,844 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=976bbe8d7afc3b1e02af11d1b515e82c, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,844 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0980fd6969c12f09b843d772c626dfa5, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,844 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d8689bfa03ba7aed23b15e5ff5d23ac, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,844 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=af843c9df2505d2eeee3c18062e9a2a6, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,844 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c0a570532487ea0ebbc51cbec9ff7e37, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,845 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=37d131a6ac713be02dc73a29079210f5, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,845 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c0f43810347933724c9e6319b11ae5cc, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,845 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9c426833151f173ef0d81d9c0ae11767, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,845 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5d3306d206c48a1c7060e49d4fe00ad5, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,845 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aa3dca7b6c799eb77565fb0e20ef1c15, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,845 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=75c927364905f2c6bffff16328d8aaf5, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,845 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d2b1eb41dda894f4f7b826a2826a9c7b, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,846 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=327d2bdcc20803fcfd1e61553f30068b, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,846 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3fe1b6a546d269c366ae3f13263e1a2a, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,846 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=324bdaae20264b3880845c3082304e57, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,846 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c66f402dd9e22edaa5996a3b7446808c, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,846 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=856730db89c49e65ccb40b5967ccf99d, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,846 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fa4e20ed49e59ee4cf63b6d84515872f, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,846 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1da36e78b320d37341113b4185c91b26, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,846 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3f148a9f8c5bea63f4744afb36553ef7, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,846 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=80bacdcc81a44a51d622e9d873892ea3, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,846 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=76cd5c1764feda4c992319b218572cef, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,847 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8cd2040dc039667d68bc1a24f5f2322f, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,847 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1cf1ce3ee0822ff957b8ffa13f56df7d, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,847 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b59f40eb016096571b13ae445f262449, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,847 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=73ec7ce1c93542e31d99b7b4474a9750, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,847 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-07T01:14:27,847 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733534127847 2024-12-07T01:14:27,847 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733534187847 2024-12-07T01:14:27,847 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-12-07T01:14:27,847 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-07T01:14:27,847 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-07T01:14:27,847 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-07T01:14:27,847 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-07T01:14:27,848 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-07T01:14:27,848 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-07T01:14:27,848 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-07T01:14:27,848 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-07T01:14:27,849 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-07T01:14:27,850 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-07T01:14:27,850 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T01:14:27,850 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T01:14:27,850 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T01:14:27,850 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-07T01:14:27,850 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T01:14:27,850 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T01:14:27,850 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T01:14:27,851 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-07T01:14:27,851 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T01:14:27,851 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T01:14:27,851 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-07T01:14:27,851 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-07T01:14:27,851 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T01:14:27,851 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-07T01:14:27,851 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-07T01:14:27,851 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-07T01:14:27,851 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-07T01:14:27,851 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-07T01:14:27,852 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 324bdaae20264b3880845c3082304e57, server=ec1863dc21e5,42915,1733534061580:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ec1863dc21e5,36079,1733534061701, splitWal=true, meta=true 2024-12-07T01:14:27,852 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ec1863dc21e5%2C36079%2C1733534061701.meta.1733534063927.meta, worker=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:27,853 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ec1863dc21e5%2C36079%2C1733534061701.meta.1733534063927.meta, worker=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:27,853 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-07T01:14:27,853 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ec1863dc21e5,40693,1733534061744 2024-12-07T01:14:27,853 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ec1863dc21e5%2C36079%2C1733534061701.1733534063448, worker=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:27,853 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ec1863dc21e5%2C36079%2C1733534061701.1733534063448, worker=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:27,853 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=37d131a6ac713be02dc73a29079210f5, ASSIGN 2024-12-07T01:14:27,853 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=324bdaae20264b3880845c3082304e57, ASSIGN 2024-12-07T01:14:27,853 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1cf1ce3ee0822ff957b8ffa13f56df7d, ASSIGN 2024-12-07T01:14:27,853 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6d8689bfa03ba7aed23b15e5ff5d23ac, ASSIGN 2024-12-07T01:14:27,853 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5d3306d206c48a1c7060e49d4fe00ad5, ASSIGN 2024-12-07T01:14:27,853 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=80bacdcc81a44a51d622e9d873892ea3, ASSIGN 2024-12-07T01:14:27,853 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d2b1eb41dda894f4f7b826a2826a9c7b, ASSIGN 2024-12-07T01:14:27,853 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fa4e20ed49e59ee4cf63b6d84515872f, ASSIGN 2024-12-07T01:14:27,853 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 80bacdcc81a44a51d622e9d873892ea3, server=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:27,853 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1cf1ce3ee0822ff957b8ffa13f56df7d, server=ec1863dc21e5,40693,1733534061744 2024-12-07T01:14:27,853 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 324bdaae20264b3880845c3082304e57, server=ec1863dc21e5,42915,1733534061580:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ec1863dc21e5,36079,1733534061701, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:27,853 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ec1863dc21e5%2C36079%2C1733534061701.meta.1733534063927.meta, worker=ec1863dc21e5,42915,1733534061580 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:27,853 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ec1863dc21e5%2C36079%2C1733534061701.meta.1733534063927.meta, worker=ec1863dc21e5,42915,1733534061580 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:27,853 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:27,853 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ec1863dc21e5,40693,1733534061744 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:27,853 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ec1863dc21e5%2C36079%2C1733534061701.1733534063448, worker=ec1863dc21e5,42915,1733534061580 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:27,853 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ec1863dc21e5%2C36079%2C1733534061701.1733534063448, worker=ec1863dc21e5,42915,1733534061580 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:27,853 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=37d131a6ac713be02dc73a29079210f5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:27,853 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=37d131a6ac713be02dc73a29079210f5, ASSIGN 2024-12-07T01:14:27,853 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=324bdaae20264b3880845c3082304e57, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:27,853 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=324bdaae20264b3880845c3082304e57, ASSIGN 2024-12-07T01:14:27,853 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1cf1ce3ee0822ff957b8ffa13f56df7d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:27,853 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1cf1ce3ee0822ff957b8ffa13f56df7d, ASSIGN 2024-12-07T01:14:27,853 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6d8689bfa03ba7aed23b15e5ff5d23ac, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:27,853 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6d8689bfa03ba7aed23b15e5ff5d23ac, ASSIGN 2024-12-07T01:14:27,854 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5d3306d206c48a1c7060e49d4fe00ad5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:27,854 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5d3306d206c48a1c7060e49d4fe00ad5, ASSIGN 2024-12-07T01:14:27,854 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=80bacdcc81a44a51d622e9d873892ea3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:27,854 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=80bacdcc81a44a51d622e9d873892ea3, ASSIGN 2024-12-07T01:14:27,854 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d2b1eb41dda894f4f7b826a2826a9c7b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:27,854 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d2b1eb41dda894f4f7b826a2826a9c7b, ASSIGN 2024-12-07T01:14:27,854 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fa4e20ed49e59ee4cf63b6d84515872f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:27,854 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fa4e20ed49e59ee4cf63b6d84515872f, ASSIGN 2024-12-07T01:14:27,854 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 80bacdcc81a44a51d622e9d873892ea3, server=ec1863dc21e5,42915,1733534061580 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:27,854 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1cf1ce3ee0822ff957b8ffa13f56df7d, server=ec1863dc21e5,40693,1733534061744 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:27,854 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-12-07T01:14:27,855 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=61 2024-12-07T01:14:27,855 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=ec1863dc21e5,40693,1733534061744, table=hbase:meta, region=1588230740 2024-12-07T01:14:27,855 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ec1863dc21e5,40693,1733534061744, state=OPEN 2024-12-07T01:14:27,859 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46141 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T01:14:27,860 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46141 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T01:14:27,864 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40693-0x101ad5a899b0003, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T01:14:27,864 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46141-0x101ad5a899b0000, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T01:14:27,864 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42915-0x101ad5a899b0001, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T01:14:27,864 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T01:14:27,864 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T01:14:27,864 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T01:14:27,865 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=37d131a6ac713be02dc73a29079210f5, ASSIGN to state=OFFLINE, location=null, table=test, region=37d131a6ac713be02dc73a29079210f5 to restore RIT 2024-12-07T01:14:27,865 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=324bdaae20264b3880845c3082304e57, ASSIGN to state=OFFLINE, location=null, table=test, region=324bdaae20264b3880845c3082304e57 to restore RIT 2024-12-07T01:14:27,865 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1cf1ce3ee0822ff957b8ffa13f56df7d, ASSIGN to state=OFFLINE, location=null, table=test, region=1cf1ce3ee0822ff957b8ffa13f56df7d to restore RIT 2024-12-07T01:14:27,865 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6d8689bfa03ba7aed23b15e5ff5d23ac, ASSIGN to state=OFFLINE, location=null, table=test, region=6d8689bfa03ba7aed23b15e5ff5d23ac to restore RIT 2024-12-07T01:14:27,865 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5d3306d206c48a1c7060e49d4fe00ad5, ASSIGN to state=OFFLINE, location=null, table=test, region=5d3306d206c48a1c7060e49d4fe00ad5 to restore RIT 2024-12-07T01:14:27,865 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=80bacdcc81a44a51d622e9d873892ea3, ASSIGN to state=OFFLINE, location=null, table=test, region=80bacdcc81a44a51d622e9d873892ea3 to restore RIT 2024-12-07T01:14:27,865 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d2b1eb41dda894f4f7b826a2826a9c7b, ASSIGN to state=OFFLINE, location=null, table=test, region=d2b1eb41dda894f4f7b826a2826a9c7b to restore RIT 2024-12-07T01:14:27,865 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fa4e20ed49e59ee4cf63b6d84515872f, ASSIGN to state=OFFLINE, location=null, table=test, region=fa4e20ed49e59ee4cf63b6d84515872f to restore RIT 2024-12-07T01:14:27,865 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733534097865 2024-12-07T01:14:27,865 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-07T01:14:27,865 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 324bdaae20264b3880845c3082304e57, server=ec1863dc21e5,42915,1733534061580:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ec1863dc21e5,36079,1733534061701, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-07T01:14:27,866 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-07T01:14:27,867 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-07T01:14:27,867 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=72, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 80bacdcc81a44a51d622e9d873892ea3, server=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:27,867 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-07T01:14:27,867 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=72, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 80bacdcc81a44a51d622e9d873892ea3, server=ec1863dc21e5,42915,1733534061580 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-07T01:14:27,871 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=976bbe8d7afc3b1e02af11d1b515e82c, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,871 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0980fd6969c12f09b843d772c626dfa5, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,872 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d8689bfa03ba7aed23b15e5ff5d23ac, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,872 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=af843c9df2505d2eeee3c18062e9a2a6, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,872 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c0a570532487ea0ebbc51cbec9ff7e37, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,872 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=37d131a6ac713be02dc73a29079210f5, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,872 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c0f43810347933724c9e6319b11ae5cc, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,872 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9c426833151f173ef0d81d9c0ae11767, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,873 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5d3306d206c48a1c7060e49d4fe00ad5, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,873 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aa3dca7b6c799eb77565fb0e20ef1c15, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,873 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=75c927364905f2c6bffff16328d8aaf5, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,873 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d2b1eb41dda894f4f7b826a2826a9c7b, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,873 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=327d2bdcc20803fcfd1e61553f30068b, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,873 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3fe1b6a546d269c366ae3f13263e1a2a, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,873 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=324bdaae20264b3880845c3082304e57, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,874 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c66f402dd9e22edaa5996a3b7446808c, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,874 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=856730db89c49e65ccb40b5967ccf99d, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,874 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fa4e20ed49e59ee4cf63b6d84515872f, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,874 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1da36e78b320d37341113b4185c91b26, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,874 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3f148a9f8c5bea63f4744afb36553ef7, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,874 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=80bacdcc81a44a51d622e9d873892ea3, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,875 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=76cd5c1764feda4c992319b218572cef, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,875 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8cd2040dc039667d68bc1a24f5f2322f, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,875 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1cf1ce3ee0822ff957b8ffa13f56df7d, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,875 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b59f40eb016096571b13ae445f262449, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,875 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=73ec7ce1c93542e31d99b7b4474a9750, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,875 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-07T01:14:27,875 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733534127875 2024-12-07T01:14:27,875 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733534187875 2024-12-07T01:14:27,875 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 10 msec 2024-12-07T01:14:27,875 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-07T01:14:27,875 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-07T01:14:27,876 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-07T01:14:27,876 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-07T01:14:27,876 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-07T01:14:27,876 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-07T01:14:27,876 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-07T01:14:27,876 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-07T01:14:27,877 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-07T01:14:27,879 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-07T01:14:27,879 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T01:14:27,879 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T01:14:27,879 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T01:14:27,879 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T01:14:27,879 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T01:14:27,879 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T01:14:27,879 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-07T01:14:27,879 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T01:14:27,880 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T01:14:27,880 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-07T01:14:27,880 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-07T01:14:27,880 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T01:14:27,880 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-07T01:14:27,880 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-07T01:14:27,880 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-07T01:14:27,880 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-07T01:14:27,881 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-07T01:14:27,883 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 324bdaae20264b3880845c3082304e57, server=ec1863dc21e5,42915,1733534061580:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ec1863dc21e5,36079,1733534061701, splitWal=true, meta=true 2024-12-07T01:14:27,883 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ec1863dc21e5%2C36079%2C1733534061701.meta.1733534063927.meta, worker=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:27,883 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ec1863dc21e5%2C36079%2C1733534061701.meta.1733534063927.meta, worker=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:27,883 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-07T01:14:27,883 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ec1863dc21e5,40693,1733534061744 2024-12-07T01:14:27,883 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ec1863dc21e5%2C36079%2C1733534061701.1733534063448, worker=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:27,883 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ec1863dc21e5%2C36079%2C1733534061701.1733534063448, worker=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:27,883 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=37d131a6ac713be02dc73a29079210f5, ASSIGN 2024-12-07T01:14:27,883 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=324bdaae20264b3880845c3082304e57, ASSIGN 2024-12-07T01:14:27,883 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1cf1ce3ee0822ff957b8ffa13f56df7d, ASSIGN 2024-12-07T01:14:27,883 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6d8689bfa03ba7aed23b15e5ff5d23ac, ASSIGN 2024-12-07T01:14:27,883 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5d3306d206c48a1c7060e49d4fe00ad5, ASSIGN 2024-12-07T01:14:27,883 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=80bacdcc81a44a51d622e9d873892ea3, ASSIGN 2024-12-07T01:14:27,883 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d2b1eb41dda894f4f7b826a2826a9c7b, ASSIGN 2024-12-07T01:14:27,883 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fa4e20ed49e59ee4cf63b6d84515872f, ASSIGN 2024-12-07T01:14:27,883 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 80bacdcc81a44a51d622e9d873892ea3, server=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:27,884 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 324bdaae20264b3880845c3082304e57, server=ec1863dc21e5,42915,1733534061580:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ec1863dc21e5,36079,1733534061701, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:27,884 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ec1863dc21e5%2C36079%2C1733534061701.meta.1733534063927.meta, worker=ec1863dc21e5,42915,1733534061580 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:27,884 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ec1863dc21e5%2C36079%2C1733534061701.meta.1733534063927.meta, worker=ec1863dc21e5,42915,1733534061580 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:27,884 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:27,884 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ec1863dc21e5,40693,1733534061744 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:27,884 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ec1863dc21e5%2C36079%2C1733534061701.1733534063448, worker=ec1863dc21e5,42915,1733534061580 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:27,884 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ec1863dc21e5%2C36079%2C1733534061701.1733534063448, worker=ec1863dc21e5,42915,1733534061580 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:27,884 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=37d131a6ac713be02dc73a29079210f5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:27,884 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=37d131a6ac713be02dc73a29079210f5, ASSIGN 2024-12-07T01:14:27,884 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=324bdaae20264b3880845c3082304e57, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:27,884 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=324bdaae20264b3880845c3082304e57, ASSIGN 2024-12-07T01:14:27,884 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1cf1ce3ee0822ff957b8ffa13f56df7d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:27,884 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1cf1ce3ee0822ff957b8ffa13f56df7d, ASSIGN 2024-12-07T01:14:27,884 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6d8689bfa03ba7aed23b15e5ff5d23ac, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:27,884 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6d8689bfa03ba7aed23b15e5ff5d23ac, ASSIGN 2024-12-07T01:14:27,884 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5d3306d206c48a1c7060e49d4fe00ad5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:27,884 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5d3306d206c48a1c7060e49d4fe00ad5, ASSIGN 2024-12-07T01:14:27,884 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=80bacdcc81a44a51d622e9d873892ea3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:27,884 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=80bacdcc81a44a51d622e9d873892ea3, ASSIGN 2024-12-07T01:14:27,884 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d2b1eb41dda894f4f7b826a2826a9c7b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:27,884 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d2b1eb41dda894f4f7b826a2826a9c7b, ASSIGN 2024-12-07T01:14:27,885 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fa4e20ed49e59ee4cf63b6d84515872f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:27,885 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fa4e20ed49e59ee4cf63b6d84515872f, ASSIGN 2024-12-07T01:14:27,885 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 80bacdcc81a44a51d622e9d873892ea3, server=ec1863dc21e5,42915,1733534061580 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:27,885 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-12-07T01:14:27,885 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=61 2024-12-07T01:14:27,886 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=ec1863dc21e5,40693,1733534061744, table=hbase:meta, region=1588230740 2024-12-07T01:14:27,886 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ec1863dc21e5,40693,1733534061744, state=OPEN 2024-12-07T01:14:27,895 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40693-0x101ad5a899b0003, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T01:14:27,895 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46141-0x101ad5a899b0000, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T01:14:27,895 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42915-0x101ad5a899b0001, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T01:14:27,896 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T01:14:27,896 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T01:14:27,896 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T01:14:27,896 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=37d131a6ac713be02dc73a29079210f5, ASSIGN to state=OFFLINE, location=null, table=test, region=37d131a6ac713be02dc73a29079210f5 to restore RIT 2024-12-07T01:14:27,896 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=324bdaae20264b3880845c3082304e57, ASSIGN to state=OFFLINE, location=null, table=test, region=324bdaae20264b3880845c3082304e57 to restore RIT 2024-12-07T01:14:27,896 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1cf1ce3ee0822ff957b8ffa13f56df7d, ASSIGN to state=OFFLINE, location=null, table=test, region=1cf1ce3ee0822ff957b8ffa13f56df7d to restore RIT 2024-12-07T01:14:27,896 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6d8689bfa03ba7aed23b15e5ff5d23ac, ASSIGN to state=OFFLINE, location=null, table=test, region=6d8689bfa03ba7aed23b15e5ff5d23ac to restore RIT 2024-12-07T01:14:27,896 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5d3306d206c48a1c7060e49d4fe00ad5, ASSIGN to state=OFFLINE, location=null, table=test, region=5d3306d206c48a1c7060e49d4fe00ad5 to restore RIT 2024-12-07T01:14:27,896 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=80bacdcc81a44a51d622e9d873892ea3, ASSIGN to state=OFFLINE, location=null, table=test, region=80bacdcc81a44a51d622e9d873892ea3 to restore RIT 2024-12-07T01:14:27,897 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d2b1eb41dda894f4f7b826a2826a9c7b, ASSIGN to state=OFFLINE, location=null, table=test, region=d2b1eb41dda894f4f7b826a2826a9c7b to restore RIT 2024-12-07T01:14:27,897 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fa4e20ed49e59ee4cf63b6d84515872f, ASSIGN to state=OFFLINE, location=null, table=test, region=fa4e20ed49e59ee4cf63b6d84515872f to restore RIT 2024-12-07T01:14:27,897 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733534097897 2024-12-07T01:14:27,897 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-07T01:14:27,897 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 324bdaae20264b3880845c3082304e57, server=ec1863dc21e5,42915,1733534061580:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ec1863dc21e5,36079,1733534061701, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-07T01:14:27,898 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-07T01:14:27,899 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-07T01:14:27,899 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fa4e20ed49e59ee4cf63b6d84515872f, ASSIGN 2024-12-07T01:14:27,899 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-07T01:14:27,899 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fa4e20ed49e59ee4cf63b6d84515872f, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-07T01:14:27,902 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=976bbe8d7afc3b1e02af11d1b515e82c, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,902 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0980fd6969c12f09b843d772c626dfa5, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,903 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d8689bfa03ba7aed23b15e5ff5d23ac, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,903 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=af843c9df2505d2eeee3c18062e9a2a6, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,903 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c0a570532487ea0ebbc51cbec9ff7e37, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,903 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=37d131a6ac713be02dc73a29079210f5, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,903 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c0f43810347933724c9e6319b11ae5cc, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,903 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9c426833151f173ef0d81d9c0ae11767, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,903 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5d3306d206c48a1c7060e49d4fe00ad5, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,903 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aa3dca7b6c799eb77565fb0e20ef1c15, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,904 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=75c927364905f2c6bffff16328d8aaf5, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,904 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d2b1eb41dda894f4f7b826a2826a9c7b, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,904 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=327d2bdcc20803fcfd1e61553f30068b, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,904 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3fe1b6a546d269c366ae3f13263e1a2a, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,904 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=324bdaae20264b3880845c3082304e57, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,904 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c66f402dd9e22edaa5996a3b7446808c, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,904 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=856730db89c49e65ccb40b5967ccf99d, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,904 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fa4e20ed49e59ee4cf63b6d84515872f, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,905 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1da36e78b320d37341113b4185c91b26, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,905 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3f148a9f8c5bea63f4744afb36553ef7, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,905 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=80bacdcc81a44a51d622e9d873892ea3, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,905 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=76cd5c1764feda4c992319b218572cef, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,905 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8cd2040dc039667d68bc1a24f5f2322f, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,905 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1cf1ce3ee0822ff957b8ffa13f56df7d, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,905 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b59f40eb016096571b13ae445f262449, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,905 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=73ec7ce1c93542e31d99b7b4474a9750, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,905 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-07T01:14:27,905 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733534127905 2024-12-07T01:14:27,906 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733534187905 2024-12-07T01:14:27,906 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-12-07T01:14:27,906 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-07T01:14:27,906 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-07T01:14:27,906 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-07T01:14:27,906 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-07T01:14:27,906 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-07T01:14:27,906 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-07T01:14:27,906 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-07T01:14:27,906 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-07T01:14:27,907 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-07T01:14:27,908 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-07T01:14:27,908 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T01:14:27,908 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T01:14:27,908 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T01:14:27,908 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T01:14:27,908 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T01:14:27,908 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T01:14:27,908 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T01:14:27,909 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T01:14:27,909 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-07T01:14:27,909 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-07T01:14:27,909 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T01:14:27,909 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-07T01:14:27,909 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-07T01:14:27,909 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-07T01:14:27,909 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-07T01:14:27,909 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-07T01:14:27,910 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 324bdaae20264b3880845c3082304e57, server=ec1863dc21e5,42915,1733534061580:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ec1863dc21e5,36079,1733534061701, splitWal=true, meta=true 2024-12-07T01:14:27,910 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ec1863dc21e5%2C36079%2C1733534061701.meta.1733534063927.meta, worker=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:27,910 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ec1863dc21e5%2C36079%2C1733534061701.meta.1733534063927.meta, worker=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:27,910 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-07T01:14:27,910 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ec1863dc21e5,40693,1733534061744 2024-12-07T01:14:27,910 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ec1863dc21e5%2C36079%2C1733534061701.1733534063448, worker=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:27,910 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ec1863dc21e5%2C36079%2C1733534061701.1733534063448, worker=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:27,910 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=37d131a6ac713be02dc73a29079210f5, ASSIGN 2024-12-07T01:14:27,910 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=324bdaae20264b3880845c3082304e57, ASSIGN 2024-12-07T01:14:27,910 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1cf1ce3ee0822ff957b8ffa13f56df7d, ASSIGN 2024-12-07T01:14:27,910 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6d8689bfa03ba7aed23b15e5ff5d23ac, ASSIGN 2024-12-07T01:14:27,911 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5d3306d206c48a1c7060e49d4fe00ad5, ASSIGN 2024-12-07T01:14:27,911 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=80bacdcc81a44a51d622e9d873892ea3, ASSIGN 2024-12-07T01:14:27,911 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d2b1eb41dda894f4f7b826a2826a9c7b, ASSIGN 2024-12-07T01:14:27,911 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fa4e20ed49e59ee4cf63b6d84515872f, ASSIGN 2024-12-07T01:14:27,911 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 324bdaae20264b3880845c3082304e57, server=ec1863dc21e5,42915,1733534061580:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ec1863dc21e5,36079,1733534061701, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:27,911 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ec1863dc21e5%2C36079%2C1733534061701.meta.1733534063927.meta, worker=ec1863dc21e5,42915,1733534061580 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:27,911 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ec1863dc21e5%2C36079%2C1733534061701.meta.1733534063927.meta, worker=ec1863dc21e5,42915,1733534061580 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:27,911 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:27,911 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ec1863dc21e5,40693,1733534061744 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:27,911 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ec1863dc21e5%2C36079%2C1733534061701.1733534063448, worker=ec1863dc21e5,42915,1733534061580 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:27,911 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ec1863dc21e5%2C36079%2C1733534061701.1733534063448, worker=ec1863dc21e5,42915,1733534061580 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:27,911 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=37d131a6ac713be02dc73a29079210f5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:27,911 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=37d131a6ac713be02dc73a29079210f5, ASSIGN 2024-12-07T01:14:27,911 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=324bdaae20264b3880845c3082304e57, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:27,911 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=324bdaae20264b3880845c3082304e57, ASSIGN 2024-12-07T01:14:27,911 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1cf1ce3ee0822ff957b8ffa13f56df7d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:27,911 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1cf1ce3ee0822ff957b8ffa13f56df7d, ASSIGN 2024-12-07T01:14:27,911 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6d8689bfa03ba7aed23b15e5ff5d23ac, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:27,911 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6d8689bfa03ba7aed23b15e5ff5d23ac, ASSIGN 2024-12-07T01:14:27,911 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5d3306d206c48a1c7060e49d4fe00ad5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:27,911 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5d3306d206c48a1c7060e49d4fe00ad5, ASSIGN 2024-12-07T01:14:27,911 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=80bacdcc81a44a51d622e9d873892ea3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:27,911 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=80bacdcc81a44a51d622e9d873892ea3, ASSIGN 2024-12-07T01:14:27,911 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d2b1eb41dda894f4f7b826a2826a9c7b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:27,911 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d2b1eb41dda894f4f7b826a2826a9c7b, ASSIGN 2024-12-07T01:14:27,911 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fa4e20ed49e59ee4cf63b6d84515872f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:27,911 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fa4e20ed49e59ee4cf63b6d84515872f, ASSIGN 2024-12-07T01:14:27,912 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-12-07T01:14:27,912 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=61 2024-12-07T01:14:27,912 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=ec1863dc21e5,40693,1733534061744, table=hbase:meta, region=1588230740 2024-12-07T01:14:27,913 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ec1863dc21e5,40693,1733534061744, state=OPEN 2024-12-07T01:14:27,922 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40693-0x101ad5a899b0003, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T01:14:27,922 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46141-0x101ad5a899b0000, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T01:14:27,922 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42915-0x101ad5a899b0001, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T01:14:27,922 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T01:14:27,922 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T01:14:27,922 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T01:14:27,923 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=37d131a6ac713be02dc73a29079210f5, ASSIGN to state=OFFLINE, location=null, table=test, region=37d131a6ac713be02dc73a29079210f5 to restore RIT 2024-12-07T01:14:27,923 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=324bdaae20264b3880845c3082304e57, ASSIGN to state=OFFLINE, location=null, table=test, region=324bdaae20264b3880845c3082304e57 to restore RIT 2024-12-07T01:14:27,923 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1cf1ce3ee0822ff957b8ffa13f56df7d, ASSIGN to state=OFFLINE, location=null, table=test, region=1cf1ce3ee0822ff957b8ffa13f56df7d to restore RIT 2024-12-07T01:14:27,923 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6d8689bfa03ba7aed23b15e5ff5d23ac, ASSIGN to state=OFFLINE, location=null, table=test, region=6d8689bfa03ba7aed23b15e5ff5d23ac to restore RIT 2024-12-07T01:14:27,923 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5d3306d206c48a1c7060e49d4fe00ad5, ASSIGN to state=OFFLINE, location=null, table=test, region=5d3306d206c48a1c7060e49d4fe00ad5 to restore RIT 2024-12-07T01:14:27,923 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=80bacdcc81a44a51d622e9d873892ea3, ASSIGN to state=OFFLINE, location=null, table=test, region=80bacdcc81a44a51d622e9d873892ea3 to restore RIT 2024-12-07T01:14:27,923 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d2b1eb41dda894f4f7b826a2826a9c7b, ASSIGN to state=OFFLINE, location=null, table=test, region=d2b1eb41dda894f4f7b826a2826a9c7b to restore RIT 2024-12-07T01:14:27,924 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fa4e20ed49e59ee4cf63b6d84515872f, ASSIGN to state=OFFLINE, location=null, table=test, region=fa4e20ed49e59ee4cf63b6d84515872f to restore RIT 2024-12-07T01:14:27,924 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733534097924 2024-12-07T01:14:27,924 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-07T01:14:27,924 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 324bdaae20264b3880845c3082304e57, server=ec1863dc21e5,42915,1733534061580:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ec1863dc21e5,36079,1733534061701, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-07T01:14:27,925 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-07T01:14:27,926 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-07T01:14:27,926 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d2b1eb41dda894f4f7b826a2826a9c7b, ASSIGN 2024-12-07T01:14:27,926 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-07T01:14:27,926 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d2b1eb41dda894f4f7b826a2826a9c7b, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-07T01:14:27,930 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=976bbe8d7afc3b1e02af11d1b515e82c, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,930 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0980fd6969c12f09b843d772c626dfa5, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,930 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d8689bfa03ba7aed23b15e5ff5d23ac, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,930 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=af843c9df2505d2eeee3c18062e9a2a6, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,930 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c0a570532487ea0ebbc51cbec9ff7e37, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,930 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=37d131a6ac713be02dc73a29079210f5, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,931 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c0f43810347933724c9e6319b11ae5cc, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,931 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9c426833151f173ef0d81d9c0ae11767, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,931 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5d3306d206c48a1c7060e49d4fe00ad5, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,931 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aa3dca7b6c799eb77565fb0e20ef1c15, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,931 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=75c927364905f2c6bffff16328d8aaf5, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,931 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d2b1eb41dda894f4f7b826a2826a9c7b, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,931 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=327d2bdcc20803fcfd1e61553f30068b, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,932 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3fe1b6a546d269c366ae3f13263e1a2a, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,932 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=324bdaae20264b3880845c3082304e57, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,932 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c66f402dd9e22edaa5996a3b7446808c, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,932 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=856730db89c49e65ccb40b5967ccf99d, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,932 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fa4e20ed49e59ee4cf63b6d84515872f, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,932 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1da36e78b320d37341113b4185c91b26, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,932 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3f148a9f8c5bea63f4744afb36553ef7, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,932 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=80bacdcc81a44a51d622e9d873892ea3, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,933 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=76cd5c1764feda4c992319b218572cef, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,933 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8cd2040dc039667d68bc1a24f5f2322f, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,933 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1cf1ce3ee0822ff957b8ffa13f56df7d, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,933 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b59f40eb016096571b13ae445f262449, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,933 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=73ec7ce1c93542e31d99b7b4474a9750, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,933 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-07T01:14:27,933 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733534127933 2024-12-07T01:14:27,933 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733534187933 2024-12-07T01:14:27,933 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-12-07T01:14:27,933 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-07T01:14:27,933 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-07T01:14:27,933 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-07T01:14:27,933 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-07T01:14:27,934 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-07T01:14:27,934 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-07T01:14:27,934 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-07T01:14:27,934 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-07T01:14:27,935 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-07T01:14:27,937 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-07T01:14:27,937 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T01:14:27,937 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T01:14:27,937 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T01:14:27,937 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T01:14:27,937 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T01:14:27,937 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T01:14:27,937 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T01:14:27,937 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-07T01:14:27,937 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-07T01:14:27,937 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T01:14:27,937 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-07T01:14:27,937 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-07T01:14:27,937 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-07T01:14:27,937 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-07T01:14:27,938 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-07T01:14:27,939 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 324bdaae20264b3880845c3082304e57, server=ec1863dc21e5,42915,1733534061580:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ec1863dc21e5,36079,1733534061701, splitWal=true, meta=true 2024-12-07T01:14:27,939 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ec1863dc21e5%2C36079%2C1733534061701.meta.1733534063927.meta, worker=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:27,939 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ec1863dc21e5%2C36079%2C1733534061701.meta.1733534063927.meta, worker=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:27,939 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-07T01:14:27,939 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ec1863dc21e5,40693,1733534061744 2024-12-07T01:14:27,939 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ec1863dc21e5%2C36079%2C1733534061701.1733534063448, worker=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:27,939 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ec1863dc21e5%2C36079%2C1733534061701.1733534063448, worker=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:27,940 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=37d131a6ac713be02dc73a29079210f5, ASSIGN 2024-12-07T01:14:27,940 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=324bdaae20264b3880845c3082304e57, ASSIGN 2024-12-07T01:14:27,940 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1cf1ce3ee0822ff957b8ffa13f56df7d, ASSIGN 2024-12-07T01:14:27,940 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6d8689bfa03ba7aed23b15e5ff5d23ac, ASSIGN 2024-12-07T01:14:27,940 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5d3306d206c48a1c7060e49d4fe00ad5, ASSIGN 2024-12-07T01:14:27,940 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=80bacdcc81a44a51d622e9d873892ea3, ASSIGN 2024-12-07T01:14:27,940 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d2b1eb41dda894f4f7b826a2826a9c7b, ASSIGN 2024-12-07T01:14:27,940 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 324bdaae20264b3880845c3082304e57, server=ec1863dc21e5,42915,1733534061580:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ec1863dc21e5,36079,1733534061701, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:27,940 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ec1863dc21e5%2C36079%2C1733534061701.meta.1733534063927.meta, worker=ec1863dc21e5,42915,1733534061580 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:27,940 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ec1863dc21e5%2C36079%2C1733534061701.meta.1733534063927.meta, worker=ec1863dc21e5,42915,1733534061580 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:27,940 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:27,940 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ec1863dc21e5,40693,1733534061744 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:27,940 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ec1863dc21e5%2C36079%2C1733534061701.1733534063448, worker=ec1863dc21e5,42915,1733534061580 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:27,940 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ec1863dc21e5%2C36079%2C1733534061701.1733534063448, worker=ec1863dc21e5,42915,1733534061580 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:27,940 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=37d131a6ac713be02dc73a29079210f5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:27,940 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=37d131a6ac713be02dc73a29079210f5, ASSIGN 2024-12-07T01:14:27,940 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=324bdaae20264b3880845c3082304e57, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:27,940 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=324bdaae20264b3880845c3082304e57, ASSIGN 2024-12-07T01:14:27,940 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1cf1ce3ee0822ff957b8ffa13f56df7d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:27,941 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1cf1ce3ee0822ff957b8ffa13f56df7d, ASSIGN 2024-12-07T01:14:27,941 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6d8689bfa03ba7aed23b15e5ff5d23ac, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:27,941 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6d8689bfa03ba7aed23b15e5ff5d23ac, ASSIGN 2024-12-07T01:14:27,941 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5d3306d206c48a1c7060e49d4fe00ad5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:27,941 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5d3306d206c48a1c7060e49d4fe00ad5, ASSIGN 2024-12-07T01:14:27,941 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=80bacdcc81a44a51d622e9d873892ea3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:27,941 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=80bacdcc81a44a51d622e9d873892ea3, ASSIGN 2024-12-07T01:14:27,941 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d2b1eb41dda894f4f7b826a2826a9c7b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:27,941 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d2b1eb41dda894f4f7b826a2826a9c7b, ASSIGN 2024-12-07T01:14:27,941 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-12-07T01:14:27,942 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=61 2024-12-07T01:14:27,942 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=ec1863dc21e5,40693,1733534061744, table=hbase:meta, region=1588230740 2024-12-07T01:14:27,942 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ec1863dc21e5,40693,1733534061744, state=OPEN 2024-12-07T01:14:27,953 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42915-0x101ad5a899b0001, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T01:14:27,953 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46141-0x101ad5a899b0000, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T01:14:27,953 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40693-0x101ad5a899b0003, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T01:14:27,954 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T01:14:27,954 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T01:14:27,954 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T01:14:27,954 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=37d131a6ac713be02dc73a29079210f5, ASSIGN to state=OFFLINE, location=null, table=test, region=37d131a6ac713be02dc73a29079210f5 to restore RIT 2024-12-07T01:14:27,954 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=324bdaae20264b3880845c3082304e57, ASSIGN to state=OFFLINE, location=null, table=test, region=324bdaae20264b3880845c3082304e57 to restore RIT 2024-12-07T01:14:27,954 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1cf1ce3ee0822ff957b8ffa13f56df7d, ASSIGN to state=OFFLINE, location=null, table=test, region=1cf1ce3ee0822ff957b8ffa13f56df7d to restore RIT 2024-12-07T01:14:27,954 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6d8689bfa03ba7aed23b15e5ff5d23ac, ASSIGN to state=OFFLINE, location=null, table=test, region=6d8689bfa03ba7aed23b15e5ff5d23ac to restore RIT 2024-12-07T01:14:27,954 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5d3306d206c48a1c7060e49d4fe00ad5, ASSIGN to state=OFFLINE, location=null, table=test, region=5d3306d206c48a1c7060e49d4fe00ad5 to restore RIT 2024-12-07T01:14:27,954 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=80bacdcc81a44a51d622e9d873892ea3, ASSIGN to state=OFFLINE, location=null, table=test, region=80bacdcc81a44a51d622e9d873892ea3 to restore RIT 2024-12-07T01:14:27,954 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d2b1eb41dda894f4f7b826a2826a9c7b, ASSIGN to state=OFFLINE, location=null, table=test, region=d2b1eb41dda894f4f7b826a2826a9c7b to restore RIT 2024-12-07T01:14:27,955 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733534097955 2024-12-07T01:14:27,955 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-07T01:14:27,955 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 324bdaae20264b3880845c3082304e57, server=ec1863dc21e5,42915,1733534061580:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ec1863dc21e5,36079,1733534061701, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-07T01:14:27,956 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-07T01:14:27,957 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-07T01:14:27,957 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=80bacdcc81a44a51d622e9d873892ea3, ASSIGN 2024-12-07T01:14:27,957 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-07T01:14:27,957 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=80bacdcc81a44a51d622e9d873892ea3, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-07T01:14:27,960 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=976bbe8d7afc3b1e02af11d1b515e82c, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,960 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0980fd6969c12f09b843d772c626dfa5, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,960 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d8689bfa03ba7aed23b15e5ff5d23ac, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,960 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=af843c9df2505d2eeee3c18062e9a2a6, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,960 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c0a570532487ea0ebbc51cbec9ff7e37, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,960 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=37d131a6ac713be02dc73a29079210f5, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,960 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c0f43810347933724c9e6319b11ae5cc, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,960 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9c426833151f173ef0d81d9c0ae11767, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,961 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5d3306d206c48a1c7060e49d4fe00ad5, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,961 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aa3dca7b6c799eb77565fb0e20ef1c15, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,961 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=75c927364905f2c6bffff16328d8aaf5, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,961 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d2b1eb41dda894f4f7b826a2826a9c7b, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,961 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=327d2bdcc20803fcfd1e61553f30068b, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,961 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3fe1b6a546d269c366ae3f13263e1a2a, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,961 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=324bdaae20264b3880845c3082304e57, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,961 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c66f402dd9e22edaa5996a3b7446808c, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,961 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=856730db89c49e65ccb40b5967ccf99d, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,961 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fa4e20ed49e59ee4cf63b6d84515872f, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,962 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1da36e78b320d37341113b4185c91b26, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,962 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3f148a9f8c5bea63f4744afb36553ef7, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,962 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=80bacdcc81a44a51d622e9d873892ea3, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,962 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=76cd5c1764feda4c992319b218572cef, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,962 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8cd2040dc039667d68bc1a24f5f2322f, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,962 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1cf1ce3ee0822ff957b8ffa13f56df7d, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,962 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b59f40eb016096571b13ae445f262449, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,962 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=73ec7ce1c93542e31d99b7b4474a9750, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,962 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-07T01:14:27,962 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733534127962 2024-12-07T01:14:27,962 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733534187962 2024-12-07T01:14:27,962 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-12-07T01:14:27,962 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-07T01:14:27,962 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-07T01:14:27,963 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-07T01:14:27,963 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-07T01:14:27,963 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-07T01:14:27,963 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-07T01:14:27,963 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-07T01:14:27,963 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-07T01:14:27,964 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-07T01:14:27,965 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-07T01:14:27,965 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T01:14:27,965 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T01:14:27,965 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T01:14:27,965 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T01:14:27,965 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T01:14:27,965 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T01:14:27,965 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-07T01:14:27,965 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-07T01:14:27,965 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T01:14:27,965 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-07T01:14:27,966 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-07T01:14:27,966 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-07T01:14:27,966 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-07T01:14:27,966 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46141 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T01:14:27,966 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-07T01:14:27,966 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46141 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T01:14:27,967 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 324bdaae20264b3880845c3082304e57, server=ec1863dc21e5,42915,1733534061580:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ec1863dc21e5,36079,1733534061701, splitWal=true, meta=true 2024-12-07T01:14:27,967 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ec1863dc21e5%2C36079%2C1733534061701.meta.1733534063927.meta, worker=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:27,967 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ec1863dc21e5%2C36079%2C1733534061701.meta.1733534063927.meta, worker=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:27,967 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-07T01:14:27,967 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ec1863dc21e5,40693,1733534061744 2024-12-07T01:14:27,967 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ec1863dc21e5%2C36079%2C1733534061701.1733534063448, worker=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:27,967 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ec1863dc21e5%2C36079%2C1733534061701.1733534063448, worker=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:27,967 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=37d131a6ac713be02dc73a29079210f5, ASSIGN 2024-12-07T01:14:27,967 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=324bdaae20264b3880845c3082304e57, ASSIGN 2024-12-07T01:14:27,967 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1cf1ce3ee0822ff957b8ffa13f56df7d, ASSIGN 2024-12-07T01:14:27,968 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6d8689bfa03ba7aed23b15e5ff5d23ac, ASSIGN 2024-12-07T01:14:27,968 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5d3306d206c48a1c7060e49d4fe00ad5, ASSIGN 2024-12-07T01:14:27,968 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=80bacdcc81a44a51d622e9d873892ea3, ASSIGN 2024-12-07T01:14:27,968 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 324bdaae20264b3880845c3082304e57, server=ec1863dc21e5,42915,1733534061580:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ec1863dc21e5,36079,1733534061701, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:27,968 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ec1863dc21e5%2C36079%2C1733534061701.meta.1733534063927.meta, worker=ec1863dc21e5,42915,1733534061580 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:27,968 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ec1863dc21e5%2C36079%2C1733534061701.meta.1733534063927.meta, worker=ec1863dc21e5,42915,1733534061580 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:27,968 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:27,968 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ec1863dc21e5,40693,1733534061744 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:27,968 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ec1863dc21e5%2C36079%2C1733534061701.1733534063448, worker=ec1863dc21e5,42915,1733534061580 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:27,968 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ec1863dc21e5%2C36079%2C1733534061701.1733534063448, worker=ec1863dc21e5,42915,1733534061580 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:27,968 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=37d131a6ac713be02dc73a29079210f5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:27,968 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=37d131a6ac713be02dc73a29079210f5, ASSIGN 2024-12-07T01:14:27,968 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=324bdaae20264b3880845c3082304e57, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:27,968 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=324bdaae20264b3880845c3082304e57, ASSIGN 2024-12-07T01:14:27,968 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1cf1ce3ee0822ff957b8ffa13f56df7d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:27,968 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1cf1ce3ee0822ff957b8ffa13f56df7d, ASSIGN 2024-12-07T01:14:27,968 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6d8689bfa03ba7aed23b15e5ff5d23ac, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:27,968 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6d8689bfa03ba7aed23b15e5ff5d23ac, ASSIGN 2024-12-07T01:14:27,968 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5d3306d206c48a1c7060e49d4fe00ad5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:27,968 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5d3306d206c48a1c7060e49d4fe00ad5, ASSIGN 2024-12-07T01:14:27,968 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=80bacdcc81a44a51d622e9d873892ea3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:27,968 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=80bacdcc81a44a51d622e9d873892ea3, ASSIGN 2024-12-07T01:14:27,968 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-12-07T01:14:27,969 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=61 2024-12-07T01:14:27,969 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=ec1863dc21e5,40693,1733534061744, table=hbase:meta, region=1588230740 2024-12-07T01:14:27,969 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ec1863dc21e5,40693,1733534061744, state=OPEN 2024-12-07T01:14:27,980 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40693-0x101ad5a899b0003, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T01:14:27,980 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42915-0x101ad5a899b0001, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T01:14:27,980 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46141-0x101ad5a899b0000, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T01:14:27,980 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T01:14:27,980 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T01:14:27,980 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T01:14:27,980 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=37d131a6ac713be02dc73a29079210f5, ASSIGN to state=OFFLINE, location=null, table=test, region=37d131a6ac713be02dc73a29079210f5 to restore RIT 2024-12-07T01:14:27,981 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=324bdaae20264b3880845c3082304e57, ASSIGN to state=OFFLINE, location=null, table=test, region=324bdaae20264b3880845c3082304e57 to restore RIT 2024-12-07T01:14:27,981 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1cf1ce3ee0822ff957b8ffa13f56df7d, ASSIGN to state=OFFLINE, location=null, table=test, region=1cf1ce3ee0822ff957b8ffa13f56df7d to restore RIT 2024-12-07T01:14:27,981 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6d8689bfa03ba7aed23b15e5ff5d23ac, ASSIGN to state=OFFLINE, location=null, table=test, region=6d8689bfa03ba7aed23b15e5ff5d23ac to restore RIT 2024-12-07T01:14:27,981 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5d3306d206c48a1c7060e49d4fe00ad5, ASSIGN to state=OFFLINE, location=null, table=test, region=5d3306d206c48a1c7060e49d4fe00ad5 to restore RIT 2024-12-07T01:14:27,981 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=80bacdcc81a44a51d622e9d873892ea3, ASSIGN to state=OFFLINE, location=null, table=test, region=80bacdcc81a44a51d622e9d873892ea3 to restore RIT 2024-12-07T01:14:27,981 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733534097981 2024-12-07T01:14:27,981 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-07T01:14:27,981 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 324bdaae20264b3880845c3082304e57, server=ec1863dc21e5,42915,1733534061580:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ec1863dc21e5,36079,1733534061701, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-07T01:14:27,982 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-07T01:14:27,983 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-07T01:14:27,983 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5d3306d206c48a1c7060e49d4fe00ad5, ASSIGN 2024-12-07T01:14:27,983 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-07T01:14:27,983 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5d3306d206c48a1c7060e49d4fe00ad5, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-07T01:14:27,986 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=976bbe8d7afc3b1e02af11d1b515e82c, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,986 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0980fd6969c12f09b843d772c626dfa5, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,986 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d8689bfa03ba7aed23b15e5ff5d23ac, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,986 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=af843c9df2505d2eeee3c18062e9a2a6, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,986 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c0a570532487ea0ebbc51cbec9ff7e37, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,986 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=37d131a6ac713be02dc73a29079210f5, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,986 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c0f43810347933724c9e6319b11ae5cc, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,986 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9c426833151f173ef0d81d9c0ae11767, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,986 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5d3306d206c48a1c7060e49d4fe00ad5, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,987 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aa3dca7b6c799eb77565fb0e20ef1c15, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,987 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=75c927364905f2c6bffff16328d8aaf5, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,987 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d2b1eb41dda894f4f7b826a2826a9c7b, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,987 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=327d2bdcc20803fcfd1e61553f30068b, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,987 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3fe1b6a546d269c366ae3f13263e1a2a, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,987 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=324bdaae20264b3880845c3082304e57, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,987 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c66f402dd9e22edaa5996a3b7446808c, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,987 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=856730db89c49e65ccb40b5967ccf99d, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,987 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fa4e20ed49e59ee4cf63b6d84515872f, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,987 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1da36e78b320d37341113b4185c91b26, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,988 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3f148a9f8c5bea63f4744afb36553ef7, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,988 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=80bacdcc81a44a51d622e9d873892ea3, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,988 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=76cd5c1764feda4c992319b218572cef, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,988 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8cd2040dc039667d68bc1a24f5f2322f, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,988 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1cf1ce3ee0822ff957b8ffa13f56df7d, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,988 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b59f40eb016096571b13ae445f262449, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:27,988 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=73ec7ce1c93542e31d99b7b4474a9750, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:27,988 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-07T01:14:27,988 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733534127988 2024-12-07T01:14:27,988 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733534187988 2024-12-07T01:14:27,988 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-12-07T01:14:27,988 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-07T01:14:27,988 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-07T01:14:27,988 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-07T01:14:27,988 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-07T01:14:27,989 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-07T01:14:27,989 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-07T01:14:27,989 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-07T01:14:27,989 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-07T01:14:27,990 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-07T01:14:27,991 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-07T01:14:27,991 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T01:14:27,991 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T01:14:27,991 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T01:14:27,991 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T01:14:27,991 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T01:14:27,991 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-07T01:14:27,991 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-07T01:14:27,991 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T01:14:27,991 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-07T01:14:27,991 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-07T01:14:27,991 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-07T01:14:27,991 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-07T01:14:27,992 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-07T01:14:27,992 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 324bdaae20264b3880845c3082304e57, server=ec1863dc21e5,42915,1733534061580:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ec1863dc21e5,36079,1733534061701, splitWal=true, meta=true 2024-12-07T01:14:27,992 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ec1863dc21e5%2C36079%2C1733534061701.meta.1733534063927.meta, worker=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:27,993 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ec1863dc21e5%2C36079%2C1733534061701.meta.1733534063927.meta, worker=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:27,993 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-07T01:14:27,993 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ec1863dc21e5,40693,1733534061744 2024-12-07T01:14:27,993 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ec1863dc21e5%2C36079%2C1733534061701.1733534063448, worker=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:27,993 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ec1863dc21e5%2C36079%2C1733534061701.1733534063448, worker=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:27,993 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=37d131a6ac713be02dc73a29079210f5, ASSIGN 2024-12-07T01:14:27,993 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=324bdaae20264b3880845c3082304e57, ASSIGN 2024-12-07T01:14:27,993 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1cf1ce3ee0822ff957b8ffa13f56df7d, ASSIGN 2024-12-07T01:14:27,993 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6d8689bfa03ba7aed23b15e5ff5d23ac, ASSIGN 2024-12-07T01:14:27,993 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5d3306d206c48a1c7060e49d4fe00ad5, ASSIGN 2024-12-07T01:14:27,993 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 324bdaae20264b3880845c3082304e57, server=ec1863dc21e5,42915,1733534061580:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ec1863dc21e5,36079,1733534061701, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:27,993 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ec1863dc21e5%2C36079%2C1733534061701.meta.1733534063927.meta, worker=ec1863dc21e5,42915,1733534061580 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:27,993 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ec1863dc21e5%2C36079%2C1733534061701.meta.1733534063927.meta, worker=ec1863dc21e5,42915,1733534061580 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:27,993 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:27,993 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ec1863dc21e5,40693,1733534061744 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:27,993 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ec1863dc21e5%2C36079%2C1733534061701.1733534063448, worker=ec1863dc21e5,42915,1733534061580 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:27,993 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ec1863dc21e5%2C36079%2C1733534061701.1733534063448, worker=ec1863dc21e5,42915,1733534061580 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:27,993 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=37d131a6ac713be02dc73a29079210f5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:27,993 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=37d131a6ac713be02dc73a29079210f5, ASSIGN 2024-12-07T01:14:27,993 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=324bdaae20264b3880845c3082304e57, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:27,993 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=324bdaae20264b3880845c3082304e57, ASSIGN 2024-12-07T01:14:27,993 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1cf1ce3ee0822ff957b8ffa13f56df7d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:27,993 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1cf1ce3ee0822ff957b8ffa13f56df7d, ASSIGN 2024-12-07T01:14:27,993 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6d8689bfa03ba7aed23b15e5ff5d23ac, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:27,993 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6d8689bfa03ba7aed23b15e5ff5d23ac, ASSIGN 2024-12-07T01:14:27,993 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5d3306d206c48a1c7060e49d4fe00ad5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:27,993 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5d3306d206c48a1c7060e49d4fe00ad5, ASSIGN 2024-12-07T01:14:27,994 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-12-07T01:14:27,994 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=61 2024-12-07T01:14:27,995 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=ec1863dc21e5,40693,1733534061744, table=hbase:meta, region=1588230740 2024-12-07T01:14:27,995 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ec1863dc21e5,40693,1733534061744, state=OPEN 2024-12-07T01:14:28,006 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40693-0x101ad5a899b0003, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T01:14:28,006 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46141-0x101ad5a899b0000, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T01:14:28,006 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42915-0x101ad5a899b0001, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T01:14:28,006 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T01:14:28,006 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T01:14:28,007 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T01:14:28,007 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=37d131a6ac713be02dc73a29079210f5, ASSIGN to state=OFFLINE, location=null, table=test, region=37d131a6ac713be02dc73a29079210f5 to restore RIT 2024-12-07T01:14:28,007 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=324bdaae20264b3880845c3082304e57, ASSIGN to state=OFFLINE, location=null, table=test, region=324bdaae20264b3880845c3082304e57 to restore RIT 2024-12-07T01:14:28,007 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1cf1ce3ee0822ff957b8ffa13f56df7d, ASSIGN to state=OFFLINE, location=null, table=test, region=1cf1ce3ee0822ff957b8ffa13f56df7d to restore RIT 2024-12-07T01:14:28,007 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6d8689bfa03ba7aed23b15e5ff5d23ac, ASSIGN to state=OFFLINE, location=null, table=test, region=6d8689bfa03ba7aed23b15e5ff5d23ac to restore RIT 2024-12-07T01:14:28,007 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5d3306d206c48a1c7060e49d4fe00ad5, ASSIGN to state=OFFLINE, location=null, table=test, region=5d3306d206c48a1c7060e49d4fe00ad5 to restore RIT 2024-12-07T01:14:28,007 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733534098007 2024-12-07T01:14:28,008 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-07T01:14:28,008 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 324bdaae20264b3880845c3082304e57, server=ec1863dc21e5,42915,1733534061580:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ec1863dc21e5,36079,1733534061701, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-07T01:14:28,009 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-07T01:14:28,010 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-07T01:14:28,010 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6d8689bfa03ba7aed23b15e5ff5d23ac, ASSIGN 2024-12-07T01:14:28,011 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-07T01:14:28,011 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6d8689bfa03ba7aed23b15e5ff5d23ac, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-07T01:14:28,012 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=976bbe8d7afc3b1e02af11d1b515e82c, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,012 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0980fd6969c12f09b843d772c626dfa5, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,012 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d8689bfa03ba7aed23b15e5ff5d23ac, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,013 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=af843c9df2505d2eeee3c18062e9a2a6, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,013 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c0a570532487ea0ebbc51cbec9ff7e37, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,013 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=37d131a6ac713be02dc73a29079210f5, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,013 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c0f43810347933724c9e6319b11ae5cc, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,013 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9c426833151f173ef0d81d9c0ae11767, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,013 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5d3306d206c48a1c7060e49d4fe00ad5, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,013 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aa3dca7b6c799eb77565fb0e20ef1c15, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,013 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=75c927364905f2c6bffff16328d8aaf5, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,013 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d2b1eb41dda894f4f7b826a2826a9c7b, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,013 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=327d2bdcc20803fcfd1e61553f30068b, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,014 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3fe1b6a546d269c366ae3f13263e1a2a, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,014 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=324bdaae20264b3880845c3082304e57, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,014 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c66f402dd9e22edaa5996a3b7446808c, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,014 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=856730db89c49e65ccb40b5967ccf99d, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,014 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fa4e20ed49e59ee4cf63b6d84515872f, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,014 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1da36e78b320d37341113b4185c91b26, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,014 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3f148a9f8c5bea63f4744afb36553ef7, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,014 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=80bacdcc81a44a51d622e9d873892ea3, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,014 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=76cd5c1764feda4c992319b218572cef, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,014 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8cd2040dc039667d68bc1a24f5f2322f, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,014 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1cf1ce3ee0822ff957b8ffa13f56df7d, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,015 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b59f40eb016096571b13ae445f262449, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,015 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=73ec7ce1c93542e31d99b7b4474a9750, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,015 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-07T01:14:28,015 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733534128015 2024-12-07T01:14:28,015 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733534188015 2024-12-07T01:14:28,015 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-12-07T01:14:28,015 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-07T01:14:28,015 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-07T01:14:28,015 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-07T01:14:28,015 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-07T01:14:28,015 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-07T01:14:28,015 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-07T01:14:28,015 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-07T01:14:28,016 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-07T01:14:28,016 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-07T01:14:28,017 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-07T01:14:28,017 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T01:14:28,018 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T01:14:28,018 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T01:14:28,018 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T01:14:28,018 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-07T01:14:28,018 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-07T01:14:28,018 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T01:14:28,018 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-07T01:14:28,018 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-07T01:14:28,018 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-07T01:14:28,018 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-07T01:14:28,018 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-07T01:14:28,019 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 324bdaae20264b3880845c3082304e57, server=ec1863dc21e5,42915,1733534061580:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ec1863dc21e5,36079,1733534061701, splitWal=true, meta=true 2024-12-07T01:14:28,019 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ec1863dc21e5%2C36079%2C1733534061701.meta.1733534063927.meta, worker=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:28,019 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ec1863dc21e5%2C36079%2C1733534061701.meta.1733534063927.meta, worker=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:28,019 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-07T01:14:28,019 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ec1863dc21e5,40693,1733534061744 2024-12-07T01:14:28,019 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ec1863dc21e5%2C36079%2C1733534061701.1733534063448, worker=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:28,019 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ec1863dc21e5%2C36079%2C1733534061701.1733534063448, worker=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:28,019 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=37d131a6ac713be02dc73a29079210f5, ASSIGN 2024-12-07T01:14:28,019 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=324bdaae20264b3880845c3082304e57, ASSIGN 2024-12-07T01:14:28,019 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1cf1ce3ee0822ff957b8ffa13f56df7d, ASSIGN 2024-12-07T01:14:28,019 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6d8689bfa03ba7aed23b15e5ff5d23ac, ASSIGN 2024-12-07T01:14:28,019 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 324bdaae20264b3880845c3082304e57, server=ec1863dc21e5,42915,1733534061580:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ec1863dc21e5,36079,1733534061701, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:28,019 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ec1863dc21e5%2C36079%2C1733534061701.meta.1733534063927.meta, worker=ec1863dc21e5,42915,1733534061580 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:28,019 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ec1863dc21e5%2C36079%2C1733534061701.meta.1733534063927.meta, worker=ec1863dc21e5,42915,1733534061580 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:28,019 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:28,020 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ec1863dc21e5,40693,1733534061744 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:28,020 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ec1863dc21e5%2C36079%2C1733534061701.1733534063448, worker=ec1863dc21e5,42915,1733534061580 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:28,020 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ec1863dc21e5%2C36079%2C1733534061701.1733534063448, worker=ec1863dc21e5,42915,1733534061580 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:28,020 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=37d131a6ac713be02dc73a29079210f5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:28,020 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=37d131a6ac713be02dc73a29079210f5, ASSIGN 2024-12-07T01:14:28,020 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=324bdaae20264b3880845c3082304e57, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:28,020 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=324bdaae20264b3880845c3082304e57, ASSIGN 2024-12-07T01:14:28,020 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1cf1ce3ee0822ff957b8ffa13f56df7d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:28,020 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1cf1ce3ee0822ff957b8ffa13f56df7d, ASSIGN 2024-12-07T01:14:28,020 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6d8689bfa03ba7aed23b15e5ff5d23ac, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:28,020 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6d8689bfa03ba7aed23b15e5ff5d23ac, ASSIGN 2024-12-07T01:14:28,020 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-12-07T01:14:28,021 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=61 2024-12-07T01:14:28,021 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=ec1863dc21e5,40693,1733534061744, table=hbase:meta, region=1588230740 2024-12-07T01:14:28,021 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ec1863dc21e5,40693,1733534061744, state=OPEN 2024-12-07T01:14:28,032 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42915-0x101ad5a899b0001, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T01:14:28,032 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40693-0x101ad5a899b0003, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T01:14:28,032 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46141-0x101ad5a899b0000, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T01:14:28,033 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T01:14:28,033 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T01:14:28,033 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T01:14:28,033 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=37d131a6ac713be02dc73a29079210f5, ASSIGN to state=OFFLINE, location=null, table=test, region=37d131a6ac713be02dc73a29079210f5 to restore RIT 2024-12-07T01:14:28,033 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=324bdaae20264b3880845c3082304e57, ASSIGN to state=OFFLINE, location=null, table=test, region=324bdaae20264b3880845c3082304e57 to restore RIT 2024-12-07T01:14:28,033 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1cf1ce3ee0822ff957b8ffa13f56df7d, ASSIGN to state=OFFLINE, location=null, table=test, region=1cf1ce3ee0822ff957b8ffa13f56df7d to restore RIT 2024-12-07T01:14:28,033 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6d8689bfa03ba7aed23b15e5ff5d23ac, ASSIGN to state=OFFLINE, location=null, table=test, region=6d8689bfa03ba7aed23b15e5ff5d23ac to restore RIT 2024-12-07T01:14:28,034 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733534098034 2024-12-07T01:14:28,034 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-07T01:14:28,034 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 324bdaae20264b3880845c3082304e57, server=ec1863dc21e5,42915,1733534061580:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ec1863dc21e5,36079,1733534061701, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-07T01:14:28,035 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-07T01:14:28,036 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-07T01:14:28,036 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1cf1ce3ee0822ff957b8ffa13f56df7d, ASSIGN 2024-12-07T01:14:28,036 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-07T01:14:28,036 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1cf1ce3ee0822ff957b8ffa13f56df7d, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-07T01:14:28,039 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=976bbe8d7afc3b1e02af11d1b515e82c, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,039 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0980fd6969c12f09b843d772c626dfa5, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,040 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d8689bfa03ba7aed23b15e5ff5d23ac, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,040 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=af843c9df2505d2eeee3c18062e9a2a6, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,040 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c0a570532487ea0ebbc51cbec9ff7e37, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,040 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=37d131a6ac713be02dc73a29079210f5, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,040 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c0f43810347933724c9e6319b11ae5cc, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,040 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9c426833151f173ef0d81d9c0ae11767, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,041 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5d3306d206c48a1c7060e49d4fe00ad5, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,041 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aa3dca7b6c799eb77565fb0e20ef1c15, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,041 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=75c927364905f2c6bffff16328d8aaf5, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,041 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d2b1eb41dda894f4f7b826a2826a9c7b, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,041 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=327d2bdcc20803fcfd1e61553f30068b, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,041 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3fe1b6a546d269c366ae3f13263e1a2a, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,042 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=324bdaae20264b3880845c3082304e57, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,042 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c66f402dd9e22edaa5996a3b7446808c, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,042 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=856730db89c49e65ccb40b5967ccf99d, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,042 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fa4e20ed49e59ee4cf63b6d84515872f, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,042 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1da36e78b320d37341113b4185c91b26, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,042 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3f148a9f8c5bea63f4744afb36553ef7, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,042 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=80bacdcc81a44a51d622e9d873892ea3, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,042 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=76cd5c1764feda4c992319b218572cef, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,042 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8cd2040dc039667d68bc1a24f5f2322f, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,042 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1cf1ce3ee0822ff957b8ffa13f56df7d, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,042 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b59f40eb016096571b13ae445f262449, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,043 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=73ec7ce1c93542e31d99b7b4474a9750, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,043 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-07T01:14:28,043 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733534128043 2024-12-07T01:14:28,043 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733534188043 2024-12-07T01:14:28,043 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-12-07T01:14:28,043 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-07T01:14:28,043 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-07T01:14:28,043 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-07T01:14:28,043 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-07T01:14:28,043 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-07T01:14:28,043 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-07T01:14:28,043 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-07T01:14:28,043 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-07T01:14:28,044 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-07T01:14:28,045 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-07T01:14:28,045 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T01:14:28,045 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T01:14:28,045 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T01:14:28,045 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-07T01:14:28,045 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-07T01:14:28,045 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T01:14:28,046 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-07T01:14:28,046 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-07T01:14:28,046 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-07T01:14:28,046 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-07T01:14:28,046 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-07T01:14:28,047 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 324bdaae20264b3880845c3082304e57, server=ec1863dc21e5,42915,1733534061580:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ec1863dc21e5,36079,1733534061701, splitWal=true, meta=true 2024-12-07T01:14:28,047 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ec1863dc21e5%2C36079%2C1733534061701.meta.1733534063927.meta, worker=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:28,047 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ec1863dc21e5%2C36079%2C1733534061701.meta.1733534063927.meta, worker=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:28,047 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-07T01:14:28,047 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ec1863dc21e5,40693,1733534061744 2024-12-07T01:14:28,047 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ec1863dc21e5%2C36079%2C1733534061701.1733534063448, worker=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:28,047 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ec1863dc21e5%2C36079%2C1733534061701.1733534063448, worker=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:28,047 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=37d131a6ac713be02dc73a29079210f5, ASSIGN 2024-12-07T01:14:28,047 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=324bdaae20264b3880845c3082304e57, ASSIGN 2024-12-07T01:14:28,047 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1cf1ce3ee0822ff957b8ffa13f56df7d, ASSIGN 2024-12-07T01:14:28,047 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 324bdaae20264b3880845c3082304e57, server=ec1863dc21e5,42915,1733534061580:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ec1863dc21e5,36079,1733534061701, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:28,047 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ec1863dc21e5%2C36079%2C1733534061701.meta.1733534063927.meta, worker=ec1863dc21e5,42915,1733534061580 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:28,047 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ec1863dc21e5%2C36079%2C1733534061701.meta.1733534063927.meta, worker=ec1863dc21e5,42915,1733534061580 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:28,047 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:28,047 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ec1863dc21e5,40693,1733534061744 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:28,047 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ec1863dc21e5%2C36079%2C1733534061701.1733534063448, worker=ec1863dc21e5,42915,1733534061580 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:28,047 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ec1863dc21e5%2C36079%2C1733534061701.1733534063448, worker=ec1863dc21e5,42915,1733534061580 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:28,047 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=37d131a6ac713be02dc73a29079210f5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:28,048 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=37d131a6ac713be02dc73a29079210f5, ASSIGN 2024-12-07T01:14:28,048 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=324bdaae20264b3880845c3082304e57, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:28,048 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=324bdaae20264b3880845c3082304e57, ASSIGN 2024-12-07T01:14:28,048 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1cf1ce3ee0822ff957b8ffa13f56df7d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:28,048 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1cf1ce3ee0822ff957b8ffa13f56df7d, ASSIGN 2024-12-07T01:14:28,048 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-12-07T01:14:28,048 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=61 2024-12-07T01:14:28,048 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=ec1863dc21e5,40693,1733534061744, table=hbase:meta, region=1588230740 2024-12-07T01:14:28,049 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ec1863dc21e5,40693,1733534061744, state=OPEN 2024-12-07T01:14:28,059 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42915-0x101ad5a899b0001, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T01:14:28,059 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46141-0x101ad5a899b0000, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T01:14:28,059 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40693-0x101ad5a899b0003, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T01:14:28,059 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T01:14:28,059 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T01:14:28,059 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T01:14:28,059 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=37d131a6ac713be02dc73a29079210f5, ASSIGN to state=OFFLINE, location=null, table=test, region=37d131a6ac713be02dc73a29079210f5 to restore RIT 2024-12-07T01:14:28,060 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=324bdaae20264b3880845c3082304e57, ASSIGN to state=OFFLINE, location=null, table=test, region=324bdaae20264b3880845c3082304e57 to restore RIT 2024-12-07T01:14:28,060 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1cf1ce3ee0822ff957b8ffa13f56df7d, ASSIGN to state=OFFLINE, location=null, table=test, region=1cf1ce3ee0822ff957b8ffa13f56df7d to restore RIT 2024-12-07T01:14:28,060 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733534098060 2024-12-07T01:14:28,060 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-07T01:14:28,060 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 324bdaae20264b3880845c3082304e57, server=ec1863dc21e5,42915,1733534061580:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ec1863dc21e5,36079,1733534061701, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-07T01:14:28,061 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-07T01:14:28,062 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-07T01:14:28,062 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=324bdaae20264b3880845c3082304e57, ASSIGN 2024-12-07T01:14:28,062 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-07T01:14:28,062 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=324bdaae20264b3880845c3082304e57, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-07T01:14:28,064 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=976bbe8d7afc3b1e02af11d1b515e82c, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,064 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0980fd6969c12f09b843d772c626dfa5, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,064 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d8689bfa03ba7aed23b15e5ff5d23ac, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,065 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=af843c9df2505d2eeee3c18062e9a2a6, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,065 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c0a570532487ea0ebbc51cbec9ff7e37, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,065 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=37d131a6ac713be02dc73a29079210f5, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,065 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c0f43810347933724c9e6319b11ae5cc, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,065 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9c426833151f173ef0d81d9c0ae11767, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,065 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5d3306d206c48a1c7060e49d4fe00ad5, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,066 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aa3dca7b6c799eb77565fb0e20ef1c15, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,066 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=75c927364905f2c6bffff16328d8aaf5, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,066 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d2b1eb41dda894f4f7b826a2826a9c7b, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,066 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=327d2bdcc20803fcfd1e61553f30068b, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,066 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3fe1b6a546d269c366ae3f13263e1a2a, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,066 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=324bdaae20264b3880845c3082304e57, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,066 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c66f402dd9e22edaa5996a3b7446808c, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,066 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=856730db89c49e65ccb40b5967ccf99d, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,067 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fa4e20ed49e59ee4cf63b6d84515872f, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,067 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1da36e78b320d37341113b4185c91b26, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,067 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3f148a9f8c5bea63f4744afb36553ef7, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,067 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=80bacdcc81a44a51d622e9d873892ea3, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,067 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=76cd5c1764feda4c992319b218572cef, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,067 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8cd2040dc039667d68bc1a24f5f2322f, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,067 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1cf1ce3ee0822ff957b8ffa13f56df7d, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,067 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b59f40eb016096571b13ae445f262449, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,067 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=73ec7ce1c93542e31d99b7b4474a9750, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,067 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-07T01:14:28,067 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733534128067 2024-12-07T01:14:28,068 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733534188067 2024-12-07T01:14:28,068 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-12-07T01:14:28,068 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-07T01:14:28,068 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-07T01:14:28,068 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-07T01:14:28,068 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-07T01:14:28,068 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-07T01:14:28,068 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-07T01:14:28,068 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-07T01:14:28,068 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-07T01:14:28,069 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-07T01:14:28,069 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46141 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T01:14:28,070 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46141 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T01:14:28,070 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-07T01:14:28,070 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T01:14:28,070 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T01:14:28,071 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-07T01:14:28,071 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-07T01:14:28,071 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T01:14:28,071 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-07T01:14:28,071 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-07T01:14:28,071 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-07T01:14:28,071 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-07T01:14:28,071 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-07T01:14:28,072 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 324bdaae20264b3880845c3082304e57, server=ec1863dc21e5,42915,1733534061580:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ec1863dc21e5,36079,1733534061701, splitWal=true, meta=true 2024-12-07T01:14:28,072 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ec1863dc21e5%2C36079%2C1733534061701.meta.1733534063927.meta, worker=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:28,072 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ec1863dc21e5%2C36079%2C1733534061701.meta.1733534063927.meta, worker=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:28,072 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-07T01:14:28,072 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ec1863dc21e5,40693,1733534061744 2024-12-07T01:14:28,072 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ec1863dc21e5%2C36079%2C1733534061701.1733534063448, worker=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:28,073 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ec1863dc21e5%2C36079%2C1733534061701.1733534063448, worker=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:28,073 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=37d131a6ac713be02dc73a29079210f5, ASSIGN 2024-12-07T01:14:28,073 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=324bdaae20264b3880845c3082304e57, ASSIGN 2024-12-07T01:14:28,073 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 324bdaae20264b3880845c3082304e57, server=ec1863dc21e5,42915,1733534061580:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ec1863dc21e5,36079,1733534061701, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:28,073 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ec1863dc21e5%2C36079%2C1733534061701.meta.1733534063927.meta, worker=ec1863dc21e5,42915,1733534061580 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:28,073 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ec1863dc21e5%2C36079%2C1733534061701.meta.1733534063927.meta, worker=ec1863dc21e5,42915,1733534061580 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:28,073 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:28,073 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ec1863dc21e5,40693,1733534061744 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:28,073 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ec1863dc21e5%2C36079%2C1733534061701.1733534063448, worker=ec1863dc21e5,42915,1733534061580 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:28,073 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ec1863dc21e5%2C36079%2C1733534061701.1733534063448, worker=ec1863dc21e5,42915,1733534061580 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:28,073 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=37d131a6ac713be02dc73a29079210f5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:28,073 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=37d131a6ac713be02dc73a29079210f5, ASSIGN 2024-12-07T01:14:28,073 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=324bdaae20264b3880845c3082304e57, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:28,073 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=324bdaae20264b3880845c3082304e57, ASSIGN 2024-12-07T01:14:28,073 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-12-07T01:14:28,074 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=61 2024-12-07T01:14:28,075 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=ec1863dc21e5,40693,1733534061744, table=hbase:meta, region=1588230740 2024-12-07T01:14:28,075 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ec1863dc21e5,40693,1733534061744, state=OPEN 2024-12-07T01:14:28,120 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46141-0x101ad5a899b0000, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T01:14:28,120 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40693-0x101ad5a899b0003, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T01:14:28,120 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42915-0x101ad5a899b0001, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T01:14:28,120 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T01:14:28,120 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T01:14:28,120 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T01:14:28,121 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=37d131a6ac713be02dc73a29079210f5, ASSIGN to state=OFFLINE, location=null, table=test, region=37d131a6ac713be02dc73a29079210f5 to restore RIT 2024-12-07T01:14:28,121 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=324bdaae20264b3880845c3082304e57, ASSIGN to state=OFFLINE, location=null, table=test, region=324bdaae20264b3880845c3082304e57 to restore RIT 2024-12-07T01:14:28,121 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733534098121 2024-12-07T01:14:28,121 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-07T01:14:28,121 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 324bdaae20264b3880845c3082304e57, server=ec1863dc21e5,42915,1733534061580:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ec1863dc21e5,36079,1733534061701, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-07T01:14:28,123 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-07T01:14:28,124 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-07T01:14:28,124 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=37d131a6ac713be02dc73a29079210f5, ASSIGN 2024-12-07T01:14:28,124 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-07T01:14:28,124 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=37d131a6ac713be02dc73a29079210f5, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-07T01:14:28,125 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=976bbe8d7afc3b1e02af11d1b515e82c, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,125 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0980fd6969c12f09b843d772c626dfa5, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,125 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d8689bfa03ba7aed23b15e5ff5d23ac, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,125 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=af843c9df2505d2eeee3c18062e9a2a6, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,126 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c0a570532487ea0ebbc51cbec9ff7e37, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,126 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=37d131a6ac713be02dc73a29079210f5, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,126 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c0f43810347933724c9e6319b11ae5cc, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,126 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9c426833151f173ef0d81d9c0ae11767, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,126 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5d3306d206c48a1c7060e49d4fe00ad5, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,126 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aa3dca7b6c799eb77565fb0e20ef1c15, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,126 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=75c927364905f2c6bffff16328d8aaf5, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,126 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d2b1eb41dda894f4f7b826a2826a9c7b, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,126 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=327d2bdcc20803fcfd1e61553f30068b, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,126 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3fe1b6a546d269c366ae3f13263e1a2a, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,126 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=324bdaae20264b3880845c3082304e57, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,126 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c66f402dd9e22edaa5996a3b7446808c, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,127 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=856730db89c49e65ccb40b5967ccf99d, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,127 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fa4e20ed49e59ee4cf63b6d84515872f, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,127 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1da36e78b320d37341113b4185c91b26, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,127 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3f148a9f8c5bea63f4744afb36553ef7, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,127 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=80bacdcc81a44a51d622e9d873892ea3, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,127 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=76cd5c1764feda4c992319b218572cef, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,127 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8cd2040dc039667d68bc1a24f5f2322f, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,127 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1cf1ce3ee0822ff957b8ffa13f56df7d, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,127 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b59f40eb016096571b13ae445f262449, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,127 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=73ec7ce1c93542e31d99b7b4474a9750, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,127 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-07T01:14:28,127 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733534128127 2024-12-07T01:14:28,127 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733534188127 2024-12-07T01:14:28,127 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 6 msec 2024-12-07T01:14:28,127 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-07T01:14:28,127 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-07T01:14:28,128 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-07T01:14:28,128 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-07T01:14:28,128 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-07T01:14:28,128 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-07T01:14:28,128 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-07T01:14:28,128 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-07T01:14:28,129 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-07T01:14:28,130 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-07T01:14:28,130 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T01:14:28,130 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-07T01:14:28,130 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-07T01:14:28,130 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T01:14:28,130 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-07T01:14:28,130 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-07T01:14:28,130 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-07T01:14:28,130 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-07T01:14:28,131 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-07T01:14:28,131 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 324bdaae20264b3880845c3082304e57, server=ec1863dc21e5,42915,1733534061580:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ec1863dc21e5,36079,1733534061701, splitWal=true, meta=true 2024-12-07T01:14:28,132 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ec1863dc21e5%2C36079%2C1733534061701.meta.1733534063927.meta, worker=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:28,132 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ec1863dc21e5%2C36079%2C1733534061701.meta.1733534063927.meta, worker=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:28,132 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-07T01:14:28,132 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ec1863dc21e5,40693,1733534061744 2024-12-07T01:14:28,132 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ec1863dc21e5%2C36079%2C1733534061701.1733534063448, worker=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:28,132 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ec1863dc21e5%2C36079%2C1733534061701.1733534063448, worker=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:28,132 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=37d131a6ac713be02dc73a29079210f5, ASSIGN 2024-12-07T01:14:28,132 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 324bdaae20264b3880845c3082304e57, server=ec1863dc21e5,42915,1733534061580:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ec1863dc21e5,36079,1733534061701, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:28,132 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ec1863dc21e5%2C36079%2C1733534061701.meta.1733534063927.meta, worker=ec1863dc21e5,42915,1733534061580 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:28,132 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ec1863dc21e5%2C36079%2C1733534061701.meta.1733534063927.meta, worker=ec1863dc21e5,42915,1733534061580 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:28,132 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:28,132 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ec1863dc21e5,40693,1733534061744 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:28,132 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ec1863dc21e5%2C36079%2C1733534061701.1733534063448, worker=ec1863dc21e5,42915,1733534061580 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:28,132 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ec1863dc21e5%2C36079%2C1733534061701.1733534063448, worker=ec1863dc21e5,42915,1733534061580 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:28,132 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=37d131a6ac713be02dc73a29079210f5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:28,132 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=37d131a6ac713be02dc73a29079210f5, ASSIGN 2024-12-07T01:14:28,132 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-12-07T01:14:28,133 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=61 2024-12-07T01:14:28,133 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=ec1863dc21e5,40693,1733534061744, table=hbase:meta, region=1588230740 2024-12-07T01:14:28,133 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ec1863dc21e5,40693,1733534061744, state=OPEN 2024-12-07T01:14:28,154 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40693-0x101ad5a899b0003, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T01:14:28,154 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46141-0x101ad5a899b0000, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T01:14:28,154 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42915-0x101ad5a899b0001, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T01:14:28,154 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T01:14:28,154 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T01:14:28,154 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T01:14:28,154 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=37d131a6ac713be02dc73a29079210f5, ASSIGN to state=OFFLINE, location=null, table=test, region=37d131a6ac713be02dc73a29079210f5 to restore RIT 2024-12-07T01:14:28,155 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733534098155 2024-12-07T01:14:28,155 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-07T01:14:28,155 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 324bdaae20264b3880845c3082304e57, server=ec1863dc21e5,42915,1733534061580:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ec1863dc21e5,36079,1733534061701, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-07T01:14:28,156 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-07T01:14:28,157 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-07T01:14:28,157 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ec1863dc21e5%2C36079%2C1733534061701.1733534063448, worker=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:28,157 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-07T01:14:28,157 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ec1863dc21e5%2C36079%2C1733534061701.1733534063448, worker=ec1863dc21e5,42915,1733534061580 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-07T01:14:28,159 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=976bbe8d7afc3b1e02af11d1b515e82c, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,159 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0980fd6969c12f09b843d772c626dfa5, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,159 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d8689bfa03ba7aed23b15e5ff5d23ac, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,159 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=af843c9df2505d2eeee3c18062e9a2a6, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,159 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c0a570532487ea0ebbc51cbec9ff7e37, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,159 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=37d131a6ac713be02dc73a29079210f5, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,159 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c0f43810347933724c9e6319b11ae5cc, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,159 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9c426833151f173ef0d81d9c0ae11767, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,160 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5d3306d206c48a1c7060e49d4fe00ad5, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,160 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aa3dca7b6c799eb77565fb0e20ef1c15, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,160 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=75c927364905f2c6bffff16328d8aaf5, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,160 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d2b1eb41dda894f4f7b826a2826a9c7b, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,160 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=327d2bdcc20803fcfd1e61553f30068b, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,160 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3fe1b6a546d269c366ae3f13263e1a2a, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,160 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=324bdaae20264b3880845c3082304e57, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,160 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c66f402dd9e22edaa5996a3b7446808c, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,160 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=856730db89c49e65ccb40b5967ccf99d, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,160 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fa4e20ed49e59ee4cf63b6d84515872f, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,160 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1da36e78b320d37341113b4185c91b26, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,160 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3f148a9f8c5bea63f4744afb36553ef7, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,160 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=80bacdcc81a44a51d622e9d873892ea3, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,161 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=76cd5c1764feda4c992319b218572cef, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,161 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8cd2040dc039667d68bc1a24f5f2322f, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,161 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1cf1ce3ee0822ff957b8ffa13f56df7d, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,161 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b59f40eb016096571b13ae445f262449, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,161 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=73ec7ce1c93542e31d99b7b4474a9750, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,161 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-07T01:14:28,161 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733534128161 2024-12-07T01:14:28,161 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733534188161 2024-12-07T01:14:28,161 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 6 msec 2024-12-07T01:14:28,161 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-07T01:14:28,161 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-07T01:14:28,161 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-07T01:14:28,161 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-07T01:14:28,161 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-07T01:14:28,161 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-07T01:14:28,161 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-07T01:14:28,162 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-07T01:14:28,162 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-07T01:14:28,164 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-07T01:14:28,164 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-07T01:14:28,164 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-07T01:14:28,164 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T01:14:28,164 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-07T01:14:28,164 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-07T01:14:28,164 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-07T01:14:28,164 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-07T01:14:28,164 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-07T01:14:28,165 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 324bdaae20264b3880845c3082304e57, server=ec1863dc21e5,42915,1733534061580:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ec1863dc21e5,36079,1733534061701, splitWal=true, meta=true 2024-12-07T01:14:28,165 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ec1863dc21e5%2C36079%2C1733534061701.meta.1733534063927.meta, worker=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:28,165 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ec1863dc21e5%2C36079%2C1733534061701.meta.1733534063927.meta, worker=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:28,165 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-07T01:14:28,165 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ec1863dc21e5,40693,1733534061744 2024-12-07T01:14:28,165 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ec1863dc21e5%2C36079%2C1733534061701.1733534063448, worker=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:28,166 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ec1863dc21e5%2C36079%2C1733534061701.1733534063448, worker=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:28,166 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 324bdaae20264b3880845c3082304e57, server=ec1863dc21e5,42915,1733534061580:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ec1863dc21e5,36079,1733534061701, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:28,166 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ec1863dc21e5%2C36079%2C1733534061701.meta.1733534063927.meta, worker=ec1863dc21e5,42915,1733534061580 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:28,166 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ec1863dc21e5%2C36079%2C1733534061701.meta.1733534063927.meta, worker=ec1863dc21e5,42915,1733534061580 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:28,166 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:28,166 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ec1863dc21e5,40693,1733534061744 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:28,166 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ec1863dc21e5%2C36079%2C1733534061701.1733534063448, worker=ec1863dc21e5,42915,1733534061580 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:28,166 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ec1863dc21e5%2C36079%2C1733534061701.1733534063448, worker=ec1863dc21e5,42915,1733534061580 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:28,166 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-12-07T01:14:28,167 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=61 2024-12-07T01:14:28,167 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=ec1863dc21e5,40693,1733534061744, table=hbase:meta, region=1588230740 2024-12-07T01:14:28,167 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ec1863dc21e5,40693,1733534061744, state=OPEN 2024-12-07T01:14:28,173 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46141 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T01:14:28,173 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46141 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T01:14:28,180 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42915-0x101ad5a899b0001, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T01:14:28,180 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40693-0x101ad5a899b0003, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T01:14:28,180 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46141-0x101ad5a899b0000, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T01:14:28,180 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T01:14:28,180 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T01:14:28,180 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T01:14:28,181 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733534098181 2024-12-07T01:14:28,181 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-07T01:14:28,181 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 324bdaae20264b3880845c3082304e57, server=ec1863dc21e5,42915,1733534061580:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ec1863dc21e5,36079,1733534061701, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-07T01:14:28,182 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-07T01:14:28,183 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-07T01:14:28,183 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ec1863dc21e5%2C36079%2C1733534061701.1733534063448, worker=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:28,183 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-07T01:14:28,183 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ec1863dc21e5%2C36079%2C1733534061701.1733534063448, worker=ec1863dc21e5,42915,1733534061580 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-07T01:14:28,185 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=976bbe8d7afc3b1e02af11d1b515e82c, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,185 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0980fd6969c12f09b843d772c626dfa5, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,185 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d8689bfa03ba7aed23b15e5ff5d23ac, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,185 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=af843c9df2505d2eeee3c18062e9a2a6, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,185 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c0a570532487ea0ebbc51cbec9ff7e37, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,185 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=37d131a6ac713be02dc73a29079210f5, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,185 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c0f43810347933724c9e6319b11ae5cc, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,185 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9c426833151f173ef0d81d9c0ae11767, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,185 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5d3306d206c48a1c7060e49d4fe00ad5, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,185 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aa3dca7b6c799eb77565fb0e20ef1c15, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,185 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=75c927364905f2c6bffff16328d8aaf5, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,185 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d2b1eb41dda894f4f7b826a2826a9c7b, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,186 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=327d2bdcc20803fcfd1e61553f30068b, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,186 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3fe1b6a546d269c366ae3f13263e1a2a, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,186 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=324bdaae20264b3880845c3082304e57, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,186 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c66f402dd9e22edaa5996a3b7446808c, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,186 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=856730db89c49e65ccb40b5967ccf99d, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,186 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fa4e20ed49e59ee4cf63b6d84515872f, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,186 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1da36e78b320d37341113b4185c91b26, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,186 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3f148a9f8c5bea63f4744afb36553ef7, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,186 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=80bacdcc81a44a51d622e9d873892ea3, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,186 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=76cd5c1764feda4c992319b218572cef, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,186 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8cd2040dc039667d68bc1a24f5f2322f, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,186 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1cf1ce3ee0822ff957b8ffa13f56df7d, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,187 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b59f40eb016096571b13ae445f262449, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,187 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=73ec7ce1c93542e31d99b7b4474a9750, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,187 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-07T01:14:28,187 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733534128187 2024-12-07T01:14:28,187 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733534188187 2024-12-07T01:14:28,187 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 6 msec 2024-12-07T01:14:28,187 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-07T01:14:28,187 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-07T01:14:28,187 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-07T01:14:28,187 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-07T01:14:28,187 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-07T01:14:28,187 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-07T01:14:28,187 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-07T01:14:28,187 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-07T01:14:28,188 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-07T01:14:28,189 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-07T01:14:28,189 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-07T01:14:28,189 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-07T01:14:28,189 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T01:14:28,189 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-07T01:14:28,189 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-07T01:14:28,190 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-07T01:14:28,190 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-07T01:14:28,190 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 324bdaae20264b3880845c3082304e57, server=ec1863dc21e5,42915,1733534061580:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ec1863dc21e5,36079,1733534061701, splitWal=true, meta=true 2024-12-07T01:14:28,191 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ec1863dc21e5%2C36079%2C1733534061701.meta.1733534063927.meta, worker=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:28,191 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ec1863dc21e5%2C36079%2C1733534061701.meta.1733534063927.meta, worker=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:28,191 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-07T01:14:28,191 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ec1863dc21e5,40693,1733534061744 2024-12-07T01:14:28,191 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ec1863dc21e5%2C36079%2C1733534061701.1733534063448, worker=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:28,191 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 324bdaae20264b3880845c3082304e57, server=ec1863dc21e5,42915,1733534061580:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ec1863dc21e5,36079,1733534061701, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:28,191 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ec1863dc21e5%2C36079%2C1733534061701.meta.1733534063927.meta, worker=ec1863dc21e5,42915,1733534061580 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:28,191 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ec1863dc21e5%2C36079%2C1733534061701.meta.1733534063927.meta, worker=ec1863dc21e5,42915,1733534061580 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:28,191 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:28,191 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ec1863dc21e5,40693,1733534061744 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:28,191 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ec1863dc21e5%2C36079%2C1733534061701.1733534063448, worker=ec1863dc21e5,42915,1733534061580 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:28,191 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-12-07T01:14:28,192 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=61 2024-12-07T01:14:28,192 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=ec1863dc21e5,40693,1733534061744, table=hbase:meta, region=1588230740 2024-12-07T01:14:28,192 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ec1863dc21e5,40693,1733534061744, state=OPEN 2024-12-07T01:14:28,201 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46141-0x101ad5a899b0000, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T01:14:28,201 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40693-0x101ad5a899b0003, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T01:14:28,201 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42915-0x101ad5a899b0001, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T01:14:28,201 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T01:14:28,201 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T01:14:28,201 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T01:14:28,203 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733534098203 2024-12-07T01:14:28,203 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-07T01:14:28,203 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 324bdaae20264b3880845c3082304e57, server=ec1863dc21e5,42915,1733534061580:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ec1863dc21e5,36079,1733534061701, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-07T01:14:28,205 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-07T01:14:28,205 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-07T01:14:28,206 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ec1863dc21e5,40693,1733534061744 2024-12-07T01:14:28,206 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-07T01:14:28,206 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ec1863dc21e5,40693,1733534061744 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-07T01:14:28,208 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=976bbe8d7afc3b1e02af11d1b515e82c, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,208 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0980fd6969c12f09b843d772c626dfa5, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,208 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d8689bfa03ba7aed23b15e5ff5d23ac, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,208 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=af843c9df2505d2eeee3c18062e9a2a6, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,208 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c0a570532487ea0ebbc51cbec9ff7e37, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,208 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=37d131a6ac713be02dc73a29079210f5, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,208 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c0f43810347933724c9e6319b11ae5cc, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,208 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9c426833151f173ef0d81d9c0ae11767, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,209 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5d3306d206c48a1c7060e49d4fe00ad5, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,209 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aa3dca7b6c799eb77565fb0e20ef1c15, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,209 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=75c927364905f2c6bffff16328d8aaf5, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,209 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d2b1eb41dda894f4f7b826a2826a9c7b, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,209 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=327d2bdcc20803fcfd1e61553f30068b, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,209 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3fe1b6a546d269c366ae3f13263e1a2a, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,209 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=324bdaae20264b3880845c3082304e57, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,209 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c66f402dd9e22edaa5996a3b7446808c, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,209 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=856730db89c49e65ccb40b5967ccf99d, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,209 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fa4e20ed49e59ee4cf63b6d84515872f, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,209 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1da36e78b320d37341113b4185c91b26, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,209 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3f148a9f8c5bea63f4744afb36553ef7, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,209 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=80bacdcc81a44a51d622e9d873892ea3, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,209 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=76cd5c1764feda4c992319b218572cef, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,210 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8cd2040dc039667d68bc1a24f5f2322f, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,210 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1cf1ce3ee0822ff957b8ffa13f56df7d, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,210 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b59f40eb016096571b13ae445f262449, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,210 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=73ec7ce1c93542e31d99b7b4474a9750, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,210 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-07T01:14:28,210 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733534128210 2024-12-07T01:14:28,210 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733534188210 2024-12-07T01:14:28,210 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 6 msec 2024-12-07T01:14:28,210 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-07T01:14:28,210 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-07T01:14:28,210 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-07T01:14:28,210 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-07T01:14:28,210 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-07T01:14:28,210 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-07T01:14:28,210 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-07T01:14:28,211 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-07T01:14:28,211 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-07T01:14:28,212 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-07T01:14:28,212 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-07T01:14:28,212 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-07T01:14:28,212 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T01:14:28,213 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-07T01:14:28,213 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-07T01:14:28,213 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-07T01:14:28,213 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 324bdaae20264b3880845c3082304e57, server=ec1863dc21e5,42915,1733534061580:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ec1863dc21e5,36079,1733534061701, splitWal=true, meta=true 2024-12-07T01:14:28,213 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ec1863dc21e5%2C36079%2C1733534061701.meta.1733534063927.meta, worker=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:28,213 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ec1863dc21e5%2C36079%2C1733534061701.meta.1733534063927.meta, worker=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:28,213 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-07T01:14:28,214 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ec1863dc21e5,40693,1733534061744 2024-12-07T01:14:28,214 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 324bdaae20264b3880845c3082304e57, server=ec1863dc21e5,42915,1733534061580:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ec1863dc21e5,36079,1733534061701, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:28,214 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ec1863dc21e5%2C36079%2C1733534061701.meta.1733534063927.meta, worker=ec1863dc21e5,42915,1733534061580 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:28,214 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ec1863dc21e5%2C36079%2C1733534061701.meta.1733534063927.meta, worker=ec1863dc21e5,42915,1733534061580 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:28,214 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:28,214 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ec1863dc21e5,40693,1733534061744 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:28,214 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-12-07T01:14:28,215 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=61 2024-12-07T01:14:28,215 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=ec1863dc21e5,40693,1733534061744, table=hbase:meta, region=1588230740 2024-12-07T01:14:28,215 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ec1863dc21e5,40693,1733534061744, state=OPEN 2024-12-07T01:14:28,227 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46141-0x101ad5a899b0000, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T01:14:28,227 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42915-0x101ad5a899b0001, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T01:14:28,227 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40693-0x101ad5a899b0003, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T01:14:28,227 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T01:14:28,227 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T01:14:28,227 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T01:14:28,228 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733534098228 2024-12-07T01:14:28,228 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-07T01:14:28,228 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 324bdaae20264b3880845c3082304e57, server=ec1863dc21e5,42915,1733534061580:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ec1863dc21e5,36079,1733534061701, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-07T01:14:28,229 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-07T01:14:28,230 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-07T01:14:28,230 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-07T01:14:28,230 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-07T01:14:28,230 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-07T01:14:28,233 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=976bbe8d7afc3b1e02af11d1b515e82c, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,233 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0980fd6969c12f09b843d772c626dfa5, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,233 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d8689bfa03ba7aed23b15e5ff5d23ac, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,233 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=af843c9df2505d2eeee3c18062e9a2a6, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,233 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c0a570532487ea0ebbc51cbec9ff7e37, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,233 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=37d131a6ac713be02dc73a29079210f5, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,233 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c0f43810347933724c9e6319b11ae5cc, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,233 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9c426833151f173ef0d81d9c0ae11767, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,233 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5d3306d206c48a1c7060e49d4fe00ad5, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,234 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aa3dca7b6c799eb77565fb0e20ef1c15, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,234 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=75c927364905f2c6bffff16328d8aaf5, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,234 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d2b1eb41dda894f4f7b826a2826a9c7b, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,234 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=327d2bdcc20803fcfd1e61553f30068b, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,234 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3fe1b6a546d269c366ae3f13263e1a2a, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,234 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=324bdaae20264b3880845c3082304e57, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,234 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c66f402dd9e22edaa5996a3b7446808c, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,234 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=856730db89c49e65ccb40b5967ccf99d, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,234 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fa4e20ed49e59ee4cf63b6d84515872f, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,234 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1da36e78b320d37341113b4185c91b26, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,234 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3f148a9f8c5bea63f4744afb36553ef7, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,234 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=80bacdcc81a44a51d622e9d873892ea3, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,234 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=76cd5c1764feda4c992319b218572cef, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,234 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8cd2040dc039667d68bc1a24f5f2322f, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,235 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1cf1ce3ee0822ff957b8ffa13f56df7d, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,235 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b59f40eb016096571b13ae445f262449, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,235 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=73ec7ce1c93542e31d99b7b4474a9750, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,235 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-07T01:14:28,235 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733534128235 2024-12-07T01:14:28,235 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733534188235 2024-12-07T01:14:28,235 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 6 msec 2024-12-07T01:14:28,235 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-07T01:14:28,235 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-07T01:14:28,235 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-07T01:14:28,235 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-07T01:14:28,235 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-07T01:14:28,235 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-07T01:14:28,235 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-07T01:14:28,235 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-07T01:14:28,236 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-07T01:14:28,237 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-07T01:14:28,237 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-07T01:14:28,237 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-07T01:14:28,237 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T01:14:28,238 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-07T01:14:28,238 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-07T01:14:28,238 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 324bdaae20264b3880845c3082304e57, server=ec1863dc21e5,42915,1733534061580:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ec1863dc21e5,36079,1733534061701, splitWal=true, meta=true 2024-12-07T01:14:28,238 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ec1863dc21e5%2C36079%2C1733534061701.meta.1733534063927.meta, worker=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:28,238 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ec1863dc21e5%2C36079%2C1733534061701.meta.1733534063927.meta, worker=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:28,238 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-07T01:14:28,238 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 324bdaae20264b3880845c3082304e57, server=ec1863dc21e5,42915,1733534061580:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ec1863dc21e5,36079,1733534061701, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:28,238 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ec1863dc21e5%2C36079%2C1733534061701.meta.1733534063927.meta, worker=ec1863dc21e5,42915,1733534061580 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:28,238 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ec1863dc21e5%2C36079%2C1733534061701.meta.1733534063927.meta, worker=ec1863dc21e5,42915,1733534061580 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:28,239 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:28,239 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-12-07T01:14:28,239 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=61 2024-12-07T01:14:28,239 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=ec1863dc21e5,40693,1733534061744, table=hbase:meta, region=1588230740 2024-12-07T01:14:28,239 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ec1863dc21e5,40693,1733534061744, state=OPEN 2024-12-07T01:14:28,248 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40693-0x101ad5a899b0003, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T01:14:28,248 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42915-0x101ad5a899b0001, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T01:14:28,248 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46141-0x101ad5a899b0000, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T01:14:28,249 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T01:14:28,249 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T01:14:28,249 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T01:14:28,249 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733534098249 2024-12-07T01:14:28,249 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-07T01:14:28,249 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 324bdaae20264b3880845c3082304e57, server=ec1863dc21e5,42915,1733534061580:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ec1863dc21e5,36079,1733534061701, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-07T01:14:28,250 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-07T01:14:28,251 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-07T01:14:28,251 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ec1863dc21e5%2C36079%2C1733534061701.meta.1733534063927.meta, worker=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:28,252 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-07T01:14:28,252 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ec1863dc21e5%2C36079%2C1733534061701.meta.1733534063927.meta, worker=ec1863dc21e5,42915,1733534061580 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-07T01:14:28,253 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=976bbe8d7afc3b1e02af11d1b515e82c, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,253 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0980fd6969c12f09b843d772c626dfa5, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,253 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d8689bfa03ba7aed23b15e5ff5d23ac, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,253 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=af843c9df2505d2eeee3c18062e9a2a6, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,253 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c0a570532487ea0ebbc51cbec9ff7e37, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,254 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=37d131a6ac713be02dc73a29079210f5, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,254 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c0f43810347933724c9e6319b11ae5cc, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,254 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9c426833151f173ef0d81d9c0ae11767, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,254 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5d3306d206c48a1c7060e49d4fe00ad5, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,254 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aa3dca7b6c799eb77565fb0e20ef1c15, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,254 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=75c927364905f2c6bffff16328d8aaf5, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,254 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d2b1eb41dda894f4f7b826a2826a9c7b, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,254 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=327d2bdcc20803fcfd1e61553f30068b, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,254 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3fe1b6a546d269c366ae3f13263e1a2a, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,254 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=324bdaae20264b3880845c3082304e57, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,254 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c66f402dd9e22edaa5996a3b7446808c, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,254 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=856730db89c49e65ccb40b5967ccf99d, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,254 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fa4e20ed49e59ee4cf63b6d84515872f, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,255 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1da36e78b320d37341113b4185c91b26, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,255 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3f148a9f8c5bea63f4744afb36553ef7, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,255 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=80bacdcc81a44a51d622e9d873892ea3, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,255 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=76cd5c1764feda4c992319b218572cef, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,255 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8cd2040dc039667d68bc1a24f5f2322f, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,255 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1cf1ce3ee0822ff957b8ffa13f56df7d, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,255 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b59f40eb016096571b13ae445f262449, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,255 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=73ec7ce1c93542e31d99b7b4474a9750, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,255 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-07T01:14:28,255 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733534128255 2024-12-07T01:14:28,255 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733534188255 2024-12-07T01:14:28,255 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 6 msec 2024-12-07T01:14:28,255 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-07T01:14:28,255 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-07T01:14:28,255 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-07T01:14:28,255 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-07T01:14:28,256 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-07T01:14:28,256 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-07T01:14:28,256 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-07T01:14:28,256 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-07T01:14:28,257 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-07T01:14:28,258 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-07T01:14:28,258 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-07T01:14:28,258 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-07T01:14:28,259 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-07T01:14:28,259 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-07T01:14:28,259 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 324bdaae20264b3880845c3082304e57, server=ec1863dc21e5,42915,1733534061580:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ec1863dc21e5,36079,1733534061701, splitWal=true, meta=true 2024-12-07T01:14:28,260 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ec1863dc21e5%2C36079%2C1733534061701.meta.1733534063927.meta, worker=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:28,260 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ec1863dc21e5%2C36079%2C1733534061701.meta.1733534063927.meta, worker=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:28,260 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 324bdaae20264b3880845c3082304e57, server=ec1863dc21e5,42915,1733534061580:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ec1863dc21e5,36079,1733534061701, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:28,260 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ec1863dc21e5%2C36079%2C1733534061701.meta.1733534063927.meta, worker=ec1863dc21e5,42915,1733534061580 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:28,260 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ec1863dc21e5%2C36079%2C1733534061701.meta.1733534063927.meta, worker=ec1863dc21e5,42915,1733534061580 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:28,260 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-12-07T01:14:28,261 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=61 2024-12-07T01:14:28,261 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=ec1863dc21e5,40693,1733534061744, table=hbase:meta, region=1588230740 2024-12-07T01:14:28,261 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ec1863dc21e5,40693,1733534061744, state=OPEN 2024-12-07T01:14:28,269 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40693-0x101ad5a899b0003, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T01:14:28,269 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42915-0x101ad5a899b0001, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T01:14:28,269 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46141-0x101ad5a899b0000, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T01:14:28,270 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T01:14:28,270 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T01:14:28,270 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T01:14:28,270 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733534098270 2024-12-07T01:14:28,270 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-07T01:14:28,270 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 324bdaae20264b3880845c3082304e57, server=ec1863dc21e5,42915,1733534061580:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ec1863dc21e5,36079,1733534061701, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-07T01:14:28,272 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-07T01:14:28,272 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-07T01:14:28,272 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ec1863dc21e5%2C36079%2C1733534061701.meta.1733534063927.meta, worker=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:28,272 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-07T01:14:28,272 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ec1863dc21e5%2C36079%2C1733534061701.meta.1733534063927.meta, worker=ec1863dc21e5,42915,1733534061580 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-07T01:14:28,274 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=976bbe8d7afc3b1e02af11d1b515e82c, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,275 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0980fd6969c12f09b843d772c626dfa5, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,275 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d8689bfa03ba7aed23b15e5ff5d23ac, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,275 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=af843c9df2505d2eeee3c18062e9a2a6, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,275 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c0a570532487ea0ebbc51cbec9ff7e37, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,275 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=37d131a6ac713be02dc73a29079210f5, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,275 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c0f43810347933724c9e6319b11ae5cc, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,275 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9c426833151f173ef0d81d9c0ae11767, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,275 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5d3306d206c48a1c7060e49d4fe00ad5, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,275 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aa3dca7b6c799eb77565fb0e20ef1c15, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,275 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=75c927364905f2c6bffff16328d8aaf5, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,275 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d2b1eb41dda894f4f7b826a2826a9c7b, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,275 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=327d2bdcc20803fcfd1e61553f30068b, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,275 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3fe1b6a546d269c366ae3f13263e1a2a, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,276 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=324bdaae20264b3880845c3082304e57, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,276 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c66f402dd9e22edaa5996a3b7446808c, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,276 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=856730db89c49e65ccb40b5967ccf99d, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,276 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fa4e20ed49e59ee4cf63b6d84515872f, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,276 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1da36e78b320d37341113b4185c91b26, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,276 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3f148a9f8c5bea63f4744afb36553ef7, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,276 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=80bacdcc81a44a51d622e9d873892ea3, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,276 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=76cd5c1764feda4c992319b218572cef, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,276 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8cd2040dc039667d68bc1a24f5f2322f, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,276 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1cf1ce3ee0822ff957b8ffa13f56df7d, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,276 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b59f40eb016096571b13ae445f262449, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,276 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=73ec7ce1c93542e31d99b7b4474a9750, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,276 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-07T01:14:28,276 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733534128276 2024-12-07T01:14:28,277 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733534188277 2024-12-07T01:14:28,277 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 6 msec 2024-12-07T01:14:28,277 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-07T01:14:28,277 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-07T01:14:28,277 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-07T01:14:28,277 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-07T01:14:28,277 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-07T01:14:28,277 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-07T01:14:28,277 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-07T01:14:28,277 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-07T01:14:28,278 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-07T01:14:28,279 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-07T01:14:28,279 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-07T01:14:28,279 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-07T01:14:28,280 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-07T01:14:28,280 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 324bdaae20264b3880845c3082304e57, server=ec1863dc21e5,42915,1733534061580:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ec1863dc21e5,36079,1733534061701, splitWal=true, meta=true 2024-12-07T01:14:28,280 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ec1863dc21e5%2C36079%2C1733534061701.meta.1733534063927.meta, worker=ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:28,280 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 324bdaae20264b3880845c3082304e57, server=ec1863dc21e5,42915,1733534061580:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ec1863dc21e5,36079,1733534061701, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:28,280 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ec1863dc21e5%2C36079%2C1733534061701.meta.1733534063927.meta, worker=ec1863dc21e5,42915,1733534061580 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T01:14:28,280 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-12-07T01:14:28,282 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=61 2024-12-07T01:14:28,282 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=ec1863dc21e5,40693,1733534061744, table=hbase:meta, region=1588230740 2024-12-07T01:14:28,282 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ec1863dc21e5,40693,1733534061744, state=OPEN 2024-12-07T01:14:28,282 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46141 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T01:14:28,283 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46141 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T01:14:28,290 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42915-0x101ad5a899b0001, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T01:14:28,290 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46141-0x101ad5a899b0000, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T01:14:28,290 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40693-0x101ad5a899b0003, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T01:14:28,291 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T01:14:28,291 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T01:14:28,291 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T01:14:28,291 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733534098291 2024-12-07T01:14:28,291 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-07T01:14:28,291 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 324bdaae20264b3880845c3082304e57, server=ec1863dc21e5,42915,1733534061580:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ec1863dc21e5,36079,1733534061701, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-07T01:14:28,292 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-07T01:14:28,293 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-07T01:14:28,293 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 324bdaae20264b3880845c3082304e57, server=ec1863dc21e5,42915,1733534061580:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ec1863dc21e5,36079,1733534061701, splitWal=true, meta=true 2024-12-07T01:14:28,293 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-07T01:14:28,293 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 324bdaae20264b3880845c3082304e57, server=ec1863dc21e5,42915,1733534061580:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ec1863dc21e5,36079,1733534061701, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1604) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-07T01:14:28,296 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=976bbe8d7afc3b1e02af11d1b515e82c, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,296 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0980fd6969c12f09b843d772c626dfa5, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,296 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d8689bfa03ba7aed23b15e5ff5d23ac, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,296 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=af843c9df2505d2eeee3c18062e9a2a6, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,296 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c0a570532487ea0ebbc51cbec9ff7e37, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,296 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=37d131a6ac713be02dc73a29079210f5, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,296 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c0f43810347933724c9e6319b11ae5cc, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,296 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9c426833151f173ef0d81d9c0ae11767, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,296 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5d3306d206c48a1c7060e49d4fe00ad5, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,297 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aa3dca7b6c799eb77565fb0e20ef1c15, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,297 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=75c927364905f2c6bffff16328d8aaf5, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,297 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d2b1eb41dda894f4f7b826a2826a9c7b, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,297 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=327d2bdcc20803fcfd1e61553f30068b, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,297 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3fe1b6a546d269c366ae3f13263e1a2a, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,297 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=324bdaae20264b3880845c3082304e57, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,297 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c66f402dd9e22edaa5996a3b7446808c, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,297 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=856730db89c49e65ccb40b5967ccf99d, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,297 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fa4e20ed49e59ee4cf63b6d84515872f, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,297 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1da36e78b320d37341113b4185c91b26, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,297 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3f148a9f8c5bea63f4744afb36553ef7, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,297 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=80bacdcc81a44a51d622e9d873892ea3, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,297 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=76cd5c1764feda4c992319b218572cef, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,297 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8cd2040dc039667d68bc1a24f5f2322f, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,298 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1cf1ce3ee0822ff957b8ffa13f56df7d, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,298 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b59f40eb016096571b13ae445f262449, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,298 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=73ec7ce1c93542e31d99b7b4474a9750, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,298 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-07T01:14:28,298 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733534128298 2024-12-07T01:14:28,298 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733534188298 2024-12-07T01:14:28,298 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 6 msec 2024-12-07T01:14:28,298 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-07T01:14:28,298 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-07T01:14:28,298 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-07T01:14:28,298 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-07T01:14:28,298 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-07T01:14:28,298 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-07T01:14:28,298 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-07T01:14:28,299 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-07T01:14:28,299 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-07T01:14:28,300 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-07T01:14:28,300 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-07T01:14:28,301 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-07T01:14:28,301 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 324bdaae20264b3880845c3082304e57, server=ec1863dc21e5,42915,1733534061580:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ec1863dc21e5,36079,1733534061701, splitWal=true, meta=true 2024-12-07T01:14:28,301 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 324bdaae20264b3880845c3082304e57, server=ec1863dc21e5,42915,1733534061580:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ec1863dc21e5,36079,1733534061701, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-07T01:14:28,301 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 1 msec 2024-12-07T01:14:28,302 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=61 2024-12-07T01:14:28,302 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=ec1863dc21e5,40693,1733534061744, table=hbase:meta, region=1588230740 2024-12-07T01:14:28,302 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ec1863dc21e5,40693,1733534061744, state=OPEN 2024-12-07T01:14:28,311 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40693-0x101ad5a899b0003, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T01:14:28,311 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42915-0x101ad5a899b0001, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T01:14:28,311 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46141-0x101ad5a899b0000, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T01:14:28,312 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T01:14:28,312 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T01:14:28,312 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T01:14:28,312 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733534098312 2024-12-07T01:14:28,312 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-07T01:14:28,312 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 324bdaae20264b3880845c3082304e57, server=ec1863dc21e5,42915,1733534061580:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ec1863dc21e5,36079,1733534061701, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-07T01:14:28,314 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-07T01:14:28,315 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=57, state=ROLLEDBACK, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 324bdaae20264b3880845c3082304e57, server=ec1863dc21e5,42915,1733534061580:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ec1863dc21e5,36079,1733534061701, splitWal=true, meta=true exec-time=2.5640 sec 2024-12-07T01:14:28,316 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=976bbe8d7afc3b1e02af11d1b515e82c, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,316 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0980fd6969c12f09b843d772c626dfa5, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,316 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d8689bfa03ba7aed23b15e5ff5d23ac, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,316 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=af843c9df2505d2eeee3c18062e9a2a6, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,316 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c0a570532487ea0ebbc51cbec9ff7e37, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,316 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=37d131a6ac713be02dc73a29079210f5, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,316 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c0f43810347933724c9e6319b11ae5cc, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,316 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9c426833151f173ef0d81d9c0ae11767, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,316 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5d3306d206c48a1c7060e49d4fe00ad5, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,316 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aa3dca7b6c799eb77565fb0e20ef1c15, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,317 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=75c927364905f2c6bffff16328d8aaf5, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,317 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d2b1eb41dda894f4f7b826a2826a9c7b, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,317 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=327d2bdcc20803fcfd1e61553f30068b, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,317 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3fe1b6a546d269c366ae3f13263e1a2a, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,317 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=324bdaae20264b3880845c3082304e57, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,317 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c66f402dd9e22edaa5996a3b7446808c, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,317 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=856730db89c49e65ccb40b5967ccf99d, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,317 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fa4e20ed49e59ee4cf63b6d84515872f, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,317 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1da36e78b320d37341113b4185c91b26, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,317 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3f148a9f8c5bea63f4744afb36553ef7, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,317 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=80bacdcc81a44a51d622e9d873892ea3, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,317 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=76cd5c1764feda4c992319b218572cef, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,317 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8cd2040dc039667d68bc1a24f5f2322f, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,317 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1cf1ce3ee0822ff957b8ffa13f56df7d, regionState=OPENING, lastHost=ec1863dc21e5,36079,1733534061701, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,318 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b59f40eb016096571b13ae445f262449, regionState=OPEN, lastHost=ec1863dc21e5,40693,1733534061744, regionLocation=ec1863dc21e5,40693,1733534061744, openSeqNum=2 2024-12-07T01:14:28,318 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=73ec7ce1c93542e31d99b7b4474a9750, regionState=OPEN, lastHost=ec1863dc21e5,42915,1733534061580, regionLocation=ec1863dc21e5,42915,1733534061580, openSeqNum=2 2024-12-07T01:14:28,318 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-07T01:14:28,318 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733534128318 2024-12-07T01:14:28,318 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733534188318 2024-12-07T01:14:28,318 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 5 msec 2024-12-07T01:14:28,332 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: master.assignment.TestRollbackSCP#testFailAndRollback Thread=348 (was 382), OpenFileDescriptor=626 (was 649), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=443 (was 443), ProcessCount=11 (was 11), AvailableMemoryMB=266 (was 242) - AvailableMemoryMB LEAK? - 2024-12-07T01:14:28,332 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-07T01:14:28,332 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-07T01:14:28,332 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.tearDownAfterClass(TestRollbackSCP.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-07T01:14:28,332 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T01:14:28,333 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T01:14:28,333 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T01:14:28,333 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-07T01:14:28,333 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=816463240, stopped=false 2024-12-07T01:14:28,334 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=ec1863dc21e5,46141,1733534060840 2024-12-07T01:14:28,343 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46141-0x101ad5a899b0000, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-07T01:14:28,343 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40693-0x101ad5a899b0003, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-07T01:14:28,343 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42915-0x101ad5a899b0001, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-07T01:14:28,343 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46141-0x101ad5a899b0000, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T01:14:28,343 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42915-0x101ad5a899b0001, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T01:14:28,343 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-07T01:14:28,343 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40693-0x101ad5a899b0003, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T01:14:28,343 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-07T01:14:28,343 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.tearDownAfterClass(TestRollbackSCP.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-07T01:14:28,344 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T01:14:28,344 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'ec1863dc21e5,42915,1733534061580' ***** 2024-12-07T01:14:28,344 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-07T01:14:28,344 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:42915-0x101ad5a899b0001, quorum=127.0.0.1:61706, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-07T01:14:28,344 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'ec1863dc21e5,40693,1733534061744' ***** 2024-12-07T01:14:28,344 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-07T01:14:28,344 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40693-0x101ad5a899b0003, quorum=127.0.0.1:61706, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-07T01:14:28,344 INFO [RS:2;ec1863dc21e5:40693 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-07T01:14:28,344 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-07T01:14:28,344 INFO [RS:2;ec1863dc21e5:40693 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-07T01:14:28,344 INFO [RS:0;ec1863dc21e5:42915 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-07T01:14:28,344 INFO [RS:2;ec1863dc21e5:40693 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-07T01:14:28,344 INFO [RS:0;ec1863dc21e5:42915 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-07T01:14:28,344 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:46141-0x101ad5a899b0000, quorum=127.0.0.1:61706, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-07T01:14:28,344 INFO [RS:0;ec1863dc21e5:42915 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-07T01:14:28,344 INFO [RS:2;ec1863dc21e5:40693 {}] regionserver.HRegionServer(3091): Received CLOSE for 1cf1ce3ee0822ff957b8ffa13f56df7d 2024-12-07T01:14:28,344 INFO [RS:0;ec1863dc21e5:42915 {}] regionserver.HRegionServer(3091): Received CLOSE for 75c927364905f2c6bffff16328d8aaf5 2024-12-07T01:14:28,345 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-07T01:14:28,345 INFO [RS:0;ec1863dc21e5:42915 {}] regionserver.HRegionServer(3091): Received CLOSE for 73ec7ce1c93542e31d99b7b4474a9750 2024-12-07T01:14:28,345 INFO [RS:2;ec1863dc21e5:40693 {}] regionserver.HRegionServer(3091): Received CLOSE for 976bbe8d7afc3b1e02af11d1b515e82c 2024-12-07T01:14:28,345 INFO [RS:0;ec1863dc21e5:42915 {}] regionserver.HRegionServer(3091): Received CLOSE for 6d8689bfa03ba7aed23b15e5ff5d23ac 2024-12-07T01:14:28,345 INFO [RS:2;ec1863dc21e5:40693 {}] regionserver.HRegionServer(3091): Received CLOSE for 1da36e78b320d37341113b4185c91b26 2024-12-07T01:14:28,345 INFO [RS:0;ec1863dc21e5:42915 {}] regionserver.HRegionServer(3091): Received CLOSE for 324bdaae20264b3880845c3082304e57 2024-12-07T01:14:28,345 INFO [RS:2;ec1863dc21e5:40693 {}] regionserver.HRegionServer(3091): Received CLOSE for c0f43810347933724c9e6319b11ae5cc 2024-12-07T01:14:28,345 INFO [RS:0;ec1863dc21e5:42915 {}] regionserver.HRegionServer(3091): Received CLOSE for 80bacdcc81a44a51d622e9d873892ea3 2024-12-07T01:14:28,345 INFO [RS:0;ec1863dc21e5:42915 {}] regionserver.HRegionServer(3091): Received CLOSE for 3f148a9f8c5bea63f4744afb36553ef7 2024-12-07T01:14:28,345 INFO [RS:2;ec1863dc21e5:40693 {}] regionserver.HRegionServer(3091): Received CLOSE for d2b1eb41dda894f4f7b826a2826a9c7b 2024-12-07T01:14:28,345 INFO [RS:0;ec1863dc21e5:42915 {}] regionserver.HRegionServer(3091): Received CLOSE for 3fe1b6a546d269c366ae3f13263e1a2a 2024-12-07T01:14:28,345 INFO [RS:2;ec1863dc21e5:40693 {}] regionserver.HRegionServer(3091): Received CLOSE for 76cd5c1764feda4c992319b218572cef 2024-12-07T01:14:28,345 INFO [RS:0;ec1863dc21e5:42915 {}] regionserver.HRegionServer(3091): Received CLOSE for c0a570532487ea0ebbc51cbec9ff7e37 2024-12-07T01:14:28,345 INFO [RS:2;ec1863dc21e5:40693 {}] regionserver.HRegionServer(3091): Received CLOSE for c66f402dd9e22edaa5996a3b7446808c 2024-12-07T01:14:28,345 INFO [RS:0;ec1863dc21e5:42915 {}] regionserver.HRegionServer(3091): Received CLOSE for 856730db89c49e65ccb40b5967ccf99d 2024-12-07T01:14:28,345 INFO [RS:2;ec1863dc21e5:40693 {}] regionserver.HRegionServer(3091): Received CLOSE for 327d2bdcc20803fcfd1e61553f30068b 2024-12-07T01:14:28,345 INFO [RS:0;ec1863dc21e5:42915 {}] regionserver.HRegionServer(3091): Received CLOSE for 9c426833151f173ef0d81d9c0ae11767 2024-12-07T01:14:28,345 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 75c927364905f2c6bffff16328d8aaf5, disabling compactions & flushes 2024-12-07T01:14:28,345 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 1cf1ce3ee0822ff957b8ffa13f56df7d, disabling compactions & flushes 2024-12-07T01:14:28,345 INFO [RS:0;ec1863dc21e5:42915 {}] regionserver.HRegionServer(3091): Received CLOSE for 8cd2040dc039667d68bc1a24f5f2322f 2024-12-07T01:14:28,345 INFO [RS:2;ec1863dc21e5:40693 {}] regionserver.HRegionServer(3091): Received CLOSE for af843c9df2505d2eeee3c18062e9a2a6 2024-12-07T01:14:28,345 INFO [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,kkk,1733534064285.75c927364905f2c6bffff16328d8aaf5. 2024-12-07T01:14:28,345 INFO [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,xxx,1733534064285.1cf1ce3ee0822ff957b8ffa13f56df7d. 2024-12-07T01:14:28,346 INFO [RS:0;ec1863dc21e5:42915 {}] regionserver.HRegionServer(3091): Received CLOSE for 5d3306d206c48a1c7060e49d4fe00ad5 2024-12-07T01:14:28,346 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,kkk,1733534064285.75c927364905f2c6bffff16328d8aaf5. 2024-12-07T01:14:28,346 INFO [RS:2;ec1863dc21e5:40693 {}] regionserver.HRegionServer(3091): Received CLOSE for fa4e20ed49e59ee4cf63b6d84515872f 2024-12-07T01:14:28,346 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,xxx,1733534064285.1cf1ce3ee0822ff957b8ffa13f56df7d. 2024-12-07T01:14:28,346 INFO [RS:0;ec1863dc21e5:42915 {}] regionserver.HRegionServer(3091): Received CLOSE for 0980fd6969c12f09b843d772c626dfa5 2024-12-07T01:14:28,346 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,kkk,1733534064285.75c927364905f2c6bffff16328d8aaf5. after waiting 0 ms 2024-12-07T01:14:28,346 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,kkk,1733534064285.75c927364905f2c6bffff16328d8aaf5. 2024-12-07T01:14:28,346 INFO [RS:2;ec1863dc21e5:40693 {}] regionserver.HRegionServer(3091): Received CLOSE for b59f40eb016096571b13ae445f262449 2024-12-07T01:14:28,346 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,xxx,1733534064285.1cf1ce3ee0822ff957b8ffa13f56df7d. after waiting 0 ms 2024-12-07T01:14:28,346 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,xxx,1733534064285.1cf1ce3ee0822ff957b8ffa13f56df7d. 2024-12-07T01:14:28,346 INFO [RS:2;ec1863dc21e5:40693 {}] regionserver.HRegionServer(3091): Received CLOSE for 37d131a6ac713be02dc73a29079210f5 2024-12-07T01:14:28,346 INFO [RS:0;ec1863dc21e5:42915 {}] regionserver.HRegionServer(959): stopping server ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:28,346 INFO [RS:0;ec1863dc21e5:42915 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-07T01:14:28,346 INFO [RS:2;ec1863dc21e5:40693 {}] regionserver.HRegionServer(3091): Received CLOSE for aa3dca7b6c799eb77565fb0e20ef1c15 2024-12-07T01:14:28,346 INFO [RS:2;ec1863dc21e5:40693 {}] regionserver.HRegionServer(959): stopping server ec1863dc21e5,40693,1733534061744 2024-12-07T01:14:28,346 INFO [RS:0;ec1863dc21e5:42915 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;ec1863dc21e5:42915. 2024-12-07T01:14:28,346 INFO [RS:2;ec1863dc21e5:40693 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-07T01:14:28,346 INFO [RS:2;ec1863dc21e5:40693 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;ec1863dc21e5:40693. 2024-12-07T01:14:28,346 DEBUG [RS:0;ec1863dc21e5:42915 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-07T01:14:28,346 DEBUG [RS:0;ec1863dc21e5:42915 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T01:14:28,346 DEBUG [RS:2;ec1863dc21e5:40693 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-07T01:14:28,346 DEBUG [RS:2;ec1863dc21e5:40693 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T01:14:28,346 INFO [RS:0;ec1863dc21e5:42915 {}] regionserver.HRegionServer(1321): Waiting on 13 regions to close 2024-12-07T01:14:28,346 INFO [RS:2;ec1863dc21e5:40693 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-07T01:14:28,346 INFO [RS:2;ec1863dc21e5:40693 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-07T01:14:28,346 INFO [RS:2;ec1863dc21e5:40693 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-07T01:14:28,346 INFO [RS:2;ec1863dc21e5:40693 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-07T01:14:28,347 INFO [RS:2;ec1863dc21e5:40693 {}] regionserver.HRegionServer(1321): Waiting on 14 regions to close 2024-12-07T01:14:28,347 DEBUG [RS:0;ec1863dc21e5:42915 {}] regionserver.HRegionServer(1351): Waiting on 0980fd6969c12f09b843d772c626dfa5, 324bdaae20264b3880845c3082304e57, 3f148a9f8c5bea63f4744afb36553ef7, 3fe1b6a546d269c366ae3f13263e1a2a, 5d3306d206c48a1c7060e49d4fe00ad5, 6d8689bfa03ba7aed23b15e5ff5d23ac, 73ec7ce1c93542e31d99b7b4474a9750, 75c927364905f2c6bffff16328d8aaf5, 80bacdcc81a44a51d622e9d873892ea3, 856730db89c49e65ccb40b5967ccf99d, 8cd2040dc039667d68bc1a24f5f2322f, 9c426833151f173ef0d81d9c0ae11767, c0a570532487ea0ebbc51cbec9ff7e37 2024-12-07T01:14:28,347 DEBUG [RS:2;ec1863dc21e5:40693 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 1cf1ce3ee0822ff957b8ffa13f56df7d, 1da36e78b320d37341113b4185c91b26, 327d2bdcc20803fcfd1e61553f30068b, 37d131a6ac713be02dc73a29079210f5, 76cd5c1764feda4c992319b218572cef, 976bbe8d7afc3b1e02af11d1b515e82c, aa3dca7b6c799eb77565fb0e20ef1c15, af843c9df2505d2eeee3c18062e9a2a6, b59f40eb016096571b13ae445f262449, c0f43810347933724c9e6319b11ae5cc, c66f402dd9e22edaa5996a3b7446808c, d2b1eb41dda894f4f7b826a2826a9c7b, fa4e20ed49e59ee4cf63b6d84515872f 2024-12-07T01:14:28,347 DEBUG [RS_CLOSE_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-07T01:14:28,347 INFO [RS_CLOSE_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-07T01:14:28,347 DEBUG [RS_CLOSE_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-07T01:14:28,347 DEBUG [RS_CLOSE_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-07T01:14:28,347 DEBUG [RS_CLOSE_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-07T01:14:28,347 INFO [RS_CLOSE_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=2.66 KB heapSize=5.44 KB 2024-12-07T01:14:28,350 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/default/test/1cf1ce3ee0822ff957b8ffa13f56df7d/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-07T01:14:28,350 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/default/test/75c927364905f2c6bffff16328d8aaf5/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-07T01:14:28,352 INFO [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,kkk,1733534064285.75c927364905f2c6bffff16328d8aaf5. 2024-12-07T01:14:28,352 INFO [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,xxx,1733534064285.1cf1ce3ee0822ff957b8ffa13f56df7d. 2024-12-07T01:14:28,352 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 75c927364905f2c6bffff16328d8aaf5: Waiting for close lock at 1733534068345Running coprocessor pre-close hooks at 1733534068345Disabling compacts and flushes for region at 1733534068345Disabling writes for close at 1733534068346 (+1 ms)Writing region close event to WAL at 1733534068346Running coprocessor post-close hooks at 1733534068351 (+5 ms)Closed at 1733534068352 (+1 ms) 2024-12-07T01:14:28,352 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 1cf1ce3ee0822ff957b8ffa13f56df7d: Waiting for close lock at 1733534068345Running coprocessor pre-close hooks at 1733534068345Disabling compacts and flushes for region at 1733534068345Disabling writes for close at 1733534068346 (+1 ms)Writing region close event to WAL at 1733534068346Running coprocessor post-close hooks at 1733534068351 (+5 ms)Closed at 1733534068352 (+1 ms) 2024-12-07T01:14:28,353 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,kkk,1733534064285.75c927364905f2c6bffff16328d8aaf5. 2024-12-07T01:14:28,353 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,xxx,1733534064285.1cf1ce3ee0822ff957b8ffa13f56df7d. 2024-12-07T01:14:28,353 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 976bbe8d7afc3b1e02af11d1b515e82c, disabling compactions & flushes 2024-12-07T01:14:28,353 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 73ec7ce1c93542e31d99b7b4474a9750, disabling compactions & flushes 2024-12-07T01:14:28,353 INFO [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,zzz,1733534064285.73ec7ce1c93542e31d99b7b4474a9750. 2024-12-07T01:14:28,353 INFO [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,,1733534064285.976bbe8d7afc3b1e02af11d1b515e82c. 2024-12-07T01:14:28,353 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,,1733534064285.976bbe8d7afc3b1e02af11d1b515e82c. 2024-12-07T01:14:28,353 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,zzz,1733534064285.73ec7ce1c93542e31d99b7b4474a9750. 2024-12-07T01:14:28,353 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,,1733534064285.976bbe8d7afc3b1e02af11d1b515e82c. after waiting 0 ms 2024-12-07T01:14:28,353 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,zzz,1733534064285.73ec7ce1c93542e31d99b7b4474a9750. after waiting 0 ms 2024-12-07T01:14:28,353 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,zzz,1733534064285.73ec7ce1c93542e31d99b7b4474a9750. 2024-12-07T01:14:28,353 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,,1733534064285.976bbe8d7afc3b1e02af11d1b515e82c. 2024-12-07T01:14:28,358 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/default/test/976bbe8d7afc3b1e02af11d1b515e82c/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-07T01:14:28,358 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/default/test/73ec7ce1c93542e31d99b7b4474a9750/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-07T01:14:28,359 INFO [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,zzz,1733534064285.73ec7ce1c93542e31d99b7b4474a9750. 2024-12-07T01:14:28,359 INFO [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,,1733534064285.976bbe8d7afc3b1e02af11d1b515e82c. 2024-12-07T01:14:28,359 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 73ec7ce1c93542e31d99b7b4474a9750: Waiting for close lock at 1733534068353Running coprocessor pre-close hooks at 1733534068353Disabling compacts and flushes for region at 1733534068353Disabling writes for close at 1733534068353Writing region close event to WAL at 1733534068354 (+1 ms)Running coprocessor post-close hooks at 1733534068359 (+5 ms)Closed at 1733534068359 2024-12-07T01:14:28,359 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 976bbe8d7afc3b1e02af11d1b515e82c: Waiting for close lock at 1733534068353Running coprocessor pre-close hooks at 1733534068353Disabling compacts and flushes for region at 1733534068353Disabling writes for close at 1733534068353Writing region close event to WAL at 1733534068354 (+1 ms)Running coprocessor post-close hooks at 1733534068359 (+5 ms)Closed at 1733534068359 2024-12-07T01:14:28,360 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,zzz,1733534064285.73ec7ce1c93542e31d99b7b4474a9750. 2024-12-07T01:14:28,360 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,,1733534064285.976bbe8d7afc3b1e02af11d1b515e82c. 2024-12-07T01:14:28,360 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 6d8689bfa03ba7aed23b15e5ff5d23ac, disabling compactions & flushes 2024-12-07T01:14:28,360 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 1da36e78b320d37341113b4185c91b26, disabling compactions & flushes 2024-12-07T01:14:28,360 INFO [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ccc,1733534064285.6d8689bfa03ba7aed23b15e5ff5d23ac. 2024-12-07T01:14:28,360 INFO [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,sss,1733534064285.1da36e78b320d37341113b4185c91b26. 2024-12-07T01:14:28,360 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ccc,1733534064285.6d8689bfa03ba7aed23b15e5ff5d23ac. 2024-12-07T01:14:28,360 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ccc,1733534064285.6d8689bfa03ba7aed23b15e5ff5d23ac. after waiting 0 ms 2024-12-07T01:14:28,360 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,sss,1733534064285.1da36e78b320d37341113b4185c91b26. 2024-12-07T01:14:28,360 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ccc,1733534064285.6d8689bfa03ba7aed23b15e5ff5d23ac. 2024-12-07T01:14:28,360 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,sss,1733534064285.1da36e78b320d37341113b4185c91b26. after waiting 0 ms 2024-12-07T01:14:28,360 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,sss,1733534064285.1da36e78b320d37341113b4185c91b26. 2024-12-07T01:14:28,361 DEBUG [RS_CLOSE_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/hbase/meta/1588230740/.tmp/info/3f9333ae782a494fa8fe6684e0a96bef is 138, key is test,ccc,1733534064285.6d8689bfa03ba7aed23b15e5ff5d23ac./info:regioninfo/1733534067148/Put/seqid=0 2024-12-07T01:14:28,364 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/default/test/6d8689bfa03ba7aed23b15e5ff5d23ac/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-07T01:14:28,365 INFO [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ccc,1733534064285.6d8689bfa03ba7aed23b15e5ff5d23ac. 2024-12-07T01:14:28,365 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 6d8689bfa03ba7aed23b15e5ff5d23ac: Waiting for close lock at 1733534068360Running coprocessor pre-close hooks at 1733534068360Disabling compacts and flushes for region at 1733534068360Disabling writes for close at 1733534068360Writing region close event to WAL at 1733534068360Running coprocessor post-close hooks at 1733534068365 (+5 ms)Closed at 1733534068365 2024-12-07T01:14:28,365 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ccc,1733534064285.6d8689bfa03ba7aed23b15e5ff5d23ac. 2024-12-07T01:14:28,366 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 324bdaae20264b3880845c3082304e57, disabling compactions & flushes 2024-12-07T01:14:28,366 INFO [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ooo,1733534064285.324bdaae20264b3880845c3082304e57. 2024-12-07T01:14:28,366 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ooo,1733534064285.324bdaae20264b3880845c3082304e57. 2024-12-07T01:14:28,366 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ooo,1733534064285.324bdaae20264b3880845c3082304e57. after waiting 0 ms 2024-12-07T01:14:28,366 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ooo,1733534064285.324bdaae20264b3880845c3082304e57. 2024-12-07T01:14:28,366 INFO [regionserver/ec1863dc21e5:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-07T01:14:28,366 INFO [regionserver/ec1863dc21e5:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-07T01:14:28,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741869_1045 (size=8195) 2024-12-07T01:14:28,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43991 is added to blk_1073741869_1045 (size=8195) 2024-12-07T01:14:28,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37235 is added to blk_1073741869_1045 (size=8195) 2024-12-07T01:14:28,370 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/default/test/324bdaae20264b3880845c3082304e57/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-07T01:14:28,371 INFO [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ooo,1733534064285.324bdaae20264b3880845c3082304e57. 2024-12-07T01:14:28,371 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/default/test/1da36e78b320d37341113b4185c91b26/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-07T01:14:28,371 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 324bdaae20264b3880845c3082304e57: Waiting for close lock at 1733534068366Running coprocessor pre-close hooks at 1733534068366Disabling compacts and flushes for region at 1733534068366Disabling writes for close at 1733534068366Writing region close event to WAL at 1733534068367 (+1 ms)Running coprocessor post-close hooks at 1733534068371 (+4 ms)Closed at 1733534068371 2024-12-07T01:14:28,371 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ooo,1733534064285.324bdaae20264b3880845c3082304e57. 2024-12-07T01:14:28,371 INFO [RS_CLOSE_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.66 KB at sequenceid=71 (bloomFilter=true), to=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/hbase/meta/1588230740/.tmp/info/3f9333ae782a494fa8fe6684e0a96bef 2024-12-07T01:14:28,371 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 80bacdcc81a44a51d622e9d873892ea3, disabling compactions & flushes 2024-12-07T01:14:28,371 INFO [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,uuu,1733534064285.80bacdcc81a44a51d622e9d873892ea3. 2024-12-07T01:14:28,371 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,uuu,1733534064285.80bacdcc81a44a51d622e9d873892ea3. 2024-12-07T01:14:28,371 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,uuu,1733534064285.80bacdcc81a44a51d622e9d873892ea3. after waiting 0 ms 2024-12-07T01:14:28,371 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,uuu,1733534064285.80bacdcc81a44a51d622e9d873892ea3. 2024-12-07T01:14:28,371 INFO [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,sss,1733534064285.1da36e78b320d37341113b4185c91b26. 2024-12-07T01:14:28,371 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 1da36e78b320d37341113b4185c91b26: Waiting for close lock at 1733534068360Running coprocessor pre-close hooks at 1733534068360Disabling compacts and flushes for region at 1733534068360Disabling writes for close at 1733534068360Writing region close event to WAL at 1733534068363 (+3 ms)Running coprocessor post-close hooks at 1733534068371 (+8 ms)Closed at 1733534068371 2024-12-07T01:14:28,372 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,sss,1733534064285.1da36e78b320d37341113b4185c91b26. 2024-12-07T01:14:28,372 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing c0f43810347933724c9e6319b11ae5cc, disabling compactions & flushes 2024-12-07T01:14:28,372 INFO [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ggg,1733534064285.c0f43810347933724c9e6319b11ae5cc. 2024-12-07T01:14:28,372 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ggg,1733534064285.c0f43810347933724c9e6319b11ae5cc. 2024-12-07T01:14:28,372 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ggg,1733534064285.c0f43810347933724c9e6319b11ae5cc. after waiting 0 ms 2024-12-07T01:14:28,372 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ggg,1733534064285.c0f43810347933724c9e6319b11ae5cc. 2024-12-07T01:14:28,375 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/default/test/80bacdcc81a44a51d622e9d873892ea3/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-07T01:14:28,376 INFO [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,uuu,1733534064285.80bacdcc81a44a51d622e9d873892ea3. 2024-12-07T01:14:28,376 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 80bacdcc81a44a51d622e9d873892ea3: Waiting for close lock at 1733534068371Running coprocessor pre-close hooks at 1733534068371Disabling compacts and flushes for region at 1733534068371Disabling writes for close at 1733534068371Writing region close event to WAL at 1733534068372 (+1 ms)Running coprocessor post-close hooks at 1733534068376 (+4 ms)Closed at 1733534068376 2024-12-07T01:14:28,376 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/default/test/c0f43810347933724c9e6319b11ae5cc/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-07T01:14:28,376 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,uuu,1733534064285.80bacdcc81a44a51d622e9d873892ea3. 2024-12-07T01:14:28,377 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 3f148a9f8c5bea63f4744afb36553ef7, disabling compactions & flushes 2024-12-07T01:14:28,377 INFO [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ttt,1733534064285.3f148a9f8c5bea63f4744afb36553ef7. 2024-12-07T01:14:28,377 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ttt,1733534064285.3f148a9f8c5bea63f4744afb36553ef7. 2024-12-07T01:14:28,377 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ttt,1733534064285.3f148a9f8c5bea63f4744afb36553ef7. after waiting 0 ms 2024-12-07T01:14:28,377 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ttt,1733534064285.3f148a9f8c5bea63f4744afb36553ef7. 2024-12-07T01:14:28,377 INFO [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ggg,1733534064285.c0f43810347933724c9e6319b11ae5cc. 2024-12-07T01:14:28,377 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for c0f43810347933724c9e6319b11ae5cc: Waiting for close lock at 1733534068372Running coprocessor pre-close hooks at 1733534068372Disabling compacts and flushes for region at 1733534068372Disabling writes for close at 1733534068372Writing region close event to WAL at 1733534068372Running coprocessor post-close hooks at 1733534068377 (+5 ms)Closed at 1733534068377 2024-12-07T01:14:28,377 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ggg,1733534064285.c0f43810347933724c9e6319b11ae5cc. 2024-12-07T01:14:28,377 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing d2b1eb41dda894f4f7b826a2826a9c7b, disabling compactions & flushes 2024-12-07T01:14:28,377 INFO [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,lll,1733534064285.d2b1eb41dda894f4f7b826a2826a9c7b. 2024-12-07T01:14:28,377 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,lll,1733534064285.d2b1eb41dda894f4f7b826a2826a9c7b. 2024-12-07T01:14:28,377 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,lll,1733534064285.d2b1eb41dda894f4f7b826a2826a9c7b. after waiting 0 ms 2024-12-07T01:14:28,377 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,lll,1733534064285.d2b1eb41dda894f4f7b826a2826a9c7b. 2024-12-07T01:14:28,379 DEBUG [RS_CLOSE_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/hbase/meta/1588230740/.tmp/info/3f9333ae782a494fa8fe6684e0a96bef as hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/hbase/meta/1588230740/info/3f9333ae782a494fa8fe6684e0a96bef 2024-12-07T01:14:28,382 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/default/test/3f148a9f8c5bea63f4744afb36553ef7/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-07T01:14:28,382 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/default/test/d2b1eb41dda894f4f7b826a2826a9c7b/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-07T01:14:28,383 INFO [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,lll,1733534064285.d2b1eb41dda894f4f7b826a2826a9c7b. 2024-12-07T01:14:28,383 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for d2b1eb41dda894f4f7b826a2826a9c7b: Waiting for close lock at 1733534068377Running coprocessor pre-close hooks at 1733534068377Disabling compacts and flushes for region at 1733534068377Disabling writes for close at 1733534068377Writing region close event to WAL at 1733534068378 (+1 ms)Running coprocessor post-close hooks at 1733534068383 (+5 ms)Closed at 1733534068383 2024-12-07T01:14:28,383 INFO [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ttt,1733534064285.3f148a9f8c5bea63f4744afb36553ef7. 2024-12-07T01:14:28,383 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 3f148a9f8c5bea63f4744afb36553ef7: Waiting for close lock at 1733534068376Running coprocessor pre-close hooks at 1733534068376Disabling compacts and flushes for region at 1733534068376Disabling writes for close at 1733534068377 (+1 ms)Writing region close event to WAL at 1733534068378 (+1 ms)Running coprocessor post-close hooks at 1733534068383 (+5 ms)Closed at 1733534068383 2024-12-07T01:14:28,383 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,lll,1733534064285.d2b1eb41dda894f4f7b826a2826a9c7b. 2024-12-07T01:14:28,383 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ttt,1733534064285.3f148a9f8c5bea63f4744afb36553ef7. 2024-12-07T01:14:28,384 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 3fe1b6a546d269c366ae3f13263e1a2a, disabling compactions & flushes 2024-12-07T01:14:28,384 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 76cd5c1764feda4c992319b218572cef, disabling compactions & flushes 2024-12-07T01:14:28,384 INFO [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,nnn,1733534064285.3fe1b6a546d269c366ae3f13263e1a2a. 2024-12-07T01:14:28,384 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,nnn,1733534064285.3fe1b6a546d269c366ae3f13263e1a2a. 2024-12-07T01:14:28,384 INFO [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,vvv,1733534064285.76cd5c1764feda4c992319b218572cef. 2024-12-07T01:14:28,384 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,nnn,1733534064285.3fe1b6a546d269c366ae3f13263e1a2a. after waiting 0 ms 2024-12-07T01:14:28,384 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,nnn,1733534064285.3fe1b6a546d269c366ae3f13263e1a2a. 2024-12-07T01:14:28,384 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,vvv,1733534064285.76cd5c1764feda4c992319b218572cef. 2024-12-07T01:14:28,384 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,vvv,1733534064285.76cd5c1764feda4c992319b218572cef. after waiting 0 ms 2024-12-07T01:14:28,384 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,vvv,1733534064285.76cd5c1764feda4c992319b218572cef. 2024-12-07T01:14:28,388 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/default/test/76cd5c1764feda4c992319b218572cef/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-07T01:14:28,388 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/default/test/3fe1b6a546d269c366ae3f13263e1a2a/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-07T01:14:28,388 INFO [RS_CLOSE_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/hbase/meta/1588230740/info/3f9333ae782a494fa8fe6684e0a96bef, entries=24, sequenceid=71, filesize=8.0 K 2024-12-07T01:14:28,389 INFO [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,vvv,1733534064285.76cd5c1764feda4c992319b218572cef. 2024-12-07T01:14:28,389 INFO [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,nnn,1733534064285.3fe1b6a546d269c366ae3f13263e1a2a. 2024-12-07T01:14:28,389 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 3fe1b6a546d269c366ae3f13263e1a2a: Waiting for close lock at 1733534068384Running coprocessor pre-close hooks at 1733534068384Disabling compacts and flushes for region at 1733534068384Disabling writes for close at 1733534068384Writing region close event to WAL at 1733534068384Running coprocessor post-close hooks at 1733534068388 (+4 ms)Closed at 1733534068389 (+1 ms) 2024-12-07T01:14:28,389 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 76cd5c1764feda4c992319b218572cef: Waiting for close lock at 1733534068384Running coprocessor pre-close hooks at 1733534068384Disabling compacts and flushes for region at 1733534068384Disabling writes for close at 1733534068384Writing region close event to WAL at 1733534068384Running coprocessor post-close hooks at 1733534068388 (+4 ms)Closed at 1733534068389 (+1 ms) 2024-12-07T01:14:28,389 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,vvv,1733534064285.76cd5c1764feda4c992319b218572cef. 2024-12-07T01:14:28,389 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,nnn,1733534064285.3fe1b6a546d269c366ae3f13263e1a2a. 2024-12-07T01:14:28,389 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing c66f402dd9e22edaa5996a3b7446808c, disabling compactions & flushes 2024-12-07T01:14:28,389 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing c0a570532487ea0ebbc51cbec9ff7e37, disabling compactions & flushes 2024-12-07T01:14:28,389 INFO [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ppp,1733534064285.c66f402dd9e22edaa5996a3b7446808c. 2024-12-07T01:14:28,389 INFO [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,eee,1733534064285.c0a570532487ea0ebbc51cbec9ff7e37. 2024-12-07T01:14:28,389 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ppp,1733534064285.c66f402dd9e22edaa5996a3b7446808c. 2024-12-07T01:14:28,389 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,eee,1733534064285.c0a570532487ea0ebbc51cbec9ff7e37. 2024-12-07T01:14:28,389 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ppp,1733534064285.c66f402dd9e22edaa5996a3b7446808c. after waiting 0 ms 2024-12-07T01:14:28,389 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,eee,1733534064285.c0a570532487ea0ebbc51cbec9ff7e37. after waiting 0 ms 2024-12-07T01:14:28,389 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ppp,1733534064285.c66f402dd9e22edaa5996a3b7446808c. 2024-12-07T01:14:28,389 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,eee,1733534064285.c0a570532487ea0ebbc51cbec9ff7e37. 2024-12-07T01:14:28,390 INFO [RS_CLOSE_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~2.66 KB/2720, heapSize ~4.67 KB/4784, currentSize=0 B/0 for 1588230740 in 42ms, sequenceid=71, compaction requested=false 2024-12-07T01:14:28,393 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/default/test/c0a570532487ea0ebbc51cbec9ff7e37/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-07T01:14:28,393 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/default/test/c66f402dd9e22edaa5996a3b7446808c/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-07T01:14:28,394 INFO [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ppp,1733534064285.c66f402dd9e22edaa5996a3b7446808c. 2024-12-07T01:14:28,394 INFO [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,eee,1733534064285.c0a570532487ea0ebbc51cbec9ff7e37. 2024-12-07T01:14:28,394 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for c66f402dd9e22edaa5996a3b7446808c: Waiting for close lock at 1733534068389Running coprocessor pre-close hooks at 1733534068389Disabling compacts and flushes for region at 1733534068389Disabling writes for close at 1733534068389Writing region close event to WAL at 1733534068389Running coprocessor post-close hooks at 1733534068394 (+5 ms)Closed at 1733534068394 2024-12-07T01:14:28,394 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for c0a570532487ea0ebbc51cbec9ff7e37: Waiting for close lock at 1733534068389Running coprocessor pre-close hooks at 1733534068389Disabling compacts and flushes for region at 1733534068389Disabling writes for close at 1733534068389Writing region close event to WAL at 1733534068389Running coprocessor post-close hooks at 1733534068394 (+5 ms)Closed at 1733534068394 2024-12-07T01:14:28,394 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,eee,1733534064285.c0a570532487ea0ebbc51cbec9ff7e37. 2024-12-07T01:14:28,394 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ppp,1733534064285.c66f402dd9e22edaa5996a3b7446808c. 2024-12-07T01:14:28,394 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 856730db89c49e65ccb40b5967ccf99d, disabling compactions & flushes 2024-12-07T01:14:28,394 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 327d2bdcc20803fcfd1e61553f30068b, disabling compactions & flushes 2024-12-07T01:14:28,394 INFO [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,qqq,1733534064285.856730db89c49e65ccb40b5967ccf99d. 2024-12-07T01:14:28,394 INFO [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,mmm,1733534064285.327d2bdcc20803fcfd1e61553f30068b. 2024-12-07T01:14:28,394 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,qqq,1733534064285.856730db89c49e65ccb40b5967ccf99d. 2024-12-07T01:14:28,394 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,qqq,1733534064285.856730db89c49e65ccb40b5967ccf99d. after waiting 0 ms 2024-12-07T01:14:28,394 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,mmm,1733534064285.327d2bdcc20803fcfd1e61553f30068b. 2024-12-07T01:14:28,394 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,qqq,1733534064285.856730db89c49e65ccb40b5967ccf99d. 2024-12-07T01:14:28,394 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,mmm,1733534064285.327d2bdcc20803fcfd1e61553f30068b. after waiting 0 ms 2024-12-07T01:14:28,394 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,mmm,1733534064285.327d2bdcc20803fcfd1e61553f30068b. 2024-12-07T01:14:28,396 DEBUG [RS_CLOSE_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/hbase/meta/1588230740/recovered.edits/74.seqid, newMaxSeqId=74, maxSeqId=60 2024-12-07T01:14:28,397 INFO [regionserver/ec1863dc21e5:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-07T01:14:28,397 DEBUG [RS_CLOSE_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-07T01:14:28,397 INFO [RS_CLOSE_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-07T01:14:28,397 DEBUG [RS_CLOSE_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733534068347Running coprocessor pre-close hooks at 1733534068347Disabling compacts and flushes for region at 1733534068347Disabling writes for close at 1733534068347Obtaining lock to block concurrent updates at 1733534068347Preparing flush snapshotting stores in 1588230740 at 1733534068347Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=2720, getHeapSize=5504, getOffHeapSize=0, getCellsCount=24 at 1733534068353 (+6 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733534068355 (+2 ms)Flushing 1588230740/info: creating writer at 1733534068355Flushing 1588230740/info: appending metadata at 1733534068360 (+5 ms)Flushing 1588230740/info: closing flushed file at 1733534068360Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@38089827: reopening flushed file at 1733534068378 (+18 ms)Finished flush of dataSize ~2.66 KB/2720, heapSize ~4.67 KB/4784, currentSize=0 B/0 for 1588230740 in 42ms, sequenceid=71, compaction requested=false at 1733534068390 (+12 ms)Writing region close event to WAL at 1733534068392 (+2 ms)Running coprocessor post-close hooks at 1733534068397 (+5 ms)Closed at 1733534068397 2024-12-07T01:14:28,397 DEBUG [RS_CLOSE_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-07T01:14:28,398 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/default/test/327d2bdcc20803fcfd1e61553f30068b/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-07T01:14:28,398 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/default/test/856730db89c49e65ccb40b5967ccf99d/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-07T01:14:28,399 INFO [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,mmm,1733534064285.327d2bdcc20803fcfd1e61553f30068b. 2024-12-07T01:14:28,399 INFO [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,qqq,1733534064285.856730db89c49e65ccb40b5967ccf99d. 2024-12-07T01:14:28,399 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 327d2bdcc20803fcfd1e61553f30068b: Waiting for close lock at 1733534068394Running coprocessor pre-close hooks at 1733534068394Disabling compacts and flushes for region at 1733534068394Disabling writes for close at 1733534068394Writing region close event to WAL at 1733534068395 (+1 ms)Running coprocessor post-close hooks at 1733534068399 (+4 ms)Closed at 1733534068399 2024-12-07T01:14:28,399 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 856730db89c49e65ccb40b5967ccf99d: Waiting for close lock at 1733534068394Running coprocessor pre-close hooks at 1733534068394Disabling compacts and flushes for region at 1733534068394Disabling writes for close at 1733534068394Writing region close event to WAL at 1733534068395 (+1 ms)Running coprocessor post-close hooks at 1733534068399 (+4 ms)Closed at 1733534068399 2024-12-07T01:14:28,399 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,qqq,1733534064285.856730db89c49e65ccb40b5967ccf99d. 2024-12-07T01:14:28,399 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,mmm,1733534064285.327d2bdcc20803fcfd1e61553f30068b. 2024-12-07T01:14:28,399 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing af843c9df2505d2eeee3c18062e9a2a6, disabling compactions & flushes 2024-12-07T01:14:28,399 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 9c426833151f173ef0d81d9c0ae11767, disabling compactions & flushes 2024-12-07T01:14:28,399 INFO [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ddd,1733534064285.af843c9df2505d2eeee3c18062e9a2a6. 2024-12-07T01:14:28,399 INFO [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,hhh,1733534064285.9c426833151f173ef0d81d9c0ae11767. 2024-12-07T01:14:28,399 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ddd,1733534064285.af843c9df2505d2eeee3c18062e9a2a6. 2024-12-07T01:14:28,399 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,hhh,1733534064285.9c426833151f173ef0d81d9c0ae11767. 2024-12-07T01:14:28,399 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ddd,1733534064285.af843c9df2505d2eeee3c18062e9a2a6. after waiting 0 ms 2024-12-07T01:14:28,399 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,hhh,1733534064285.9c426833151f173ef0d81d9c0ae11767. after waiting 0 ms 2024-12-07T01:14:28,400 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,hhh,1733534064285.9c426833151f173ef0d81d9c0ae11767. 2024-12-07T01:14:28,400 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ddd,1733534064285.af843c9df2505d2eeee3c18062e9a2a6. 2024-12-07T01:14:28,402 INFO [regionserver/ec1863dc21e5:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-07T01:14:28,403 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/default/test/9c426833151f173ef0d81d9c0ae11767/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-07T01:14:28,403 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/default/test/af843c9df2505d2eeee3c18062e9a2a6/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-07T01:14:28,404 INFO [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,hhh,1733534064285.9c426833151f173ef0d81d9c0ae11767. 2024-12-07T01:14:28,404 INFO [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ddd,1733534064285.af843c9df2505d2eeee3c18062e9a2a6. 2024-12-07T01:14:28,404 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 9c426833151f173ef0d81d9c0ae11767: Waiting for close lock at 1733534068399Running coprocessor pre-close hooks at 1733534068399Disabling compacts and flushes for region at 1733534068399Disabling writes for close at 1733534068400 (+1 ms)Writing region close event to WAL at 1733534068400Running coprocessor post-close hooks at 1733534068404 (+4 ms)Closed at 1733534068404 2024-12-07T01:14:28,404 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for af843c9df2505d2eeee3c18062e9a2a6: Waiting for close lock at 1733534068399Running coprocessor pre-close hooks at 1733534068399Disabling compacts and flushes for region at 1733534068399Disabling writes for close at 1733534068400 (+1 ms)Writing region close event to WAL at 1733534068400Running coprocessor post-close hooks at 1733534068404 (+4 ms)Closed at 1733534068404 2024-12-07T01:14:28,404 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,hhh,1733534064285.9c426833151f173ef0d81d9c0ae11767. 2024-12-07T01:14:28,404 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ddd,1733534064285.af843c9df2505d2eeee3c18062e9a2a6. 2024-12-07T01:14:28,404 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 8cd2040dc039667d68bc1a24f5f2322f, disabling compactions & flushes 2024-12-07T01:14:28,404 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing fa4e20ed49e59ee4cf63b6d84515872f, disabling compactions & flushes 2024-12-07T01:14:28,404 INFO [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,www,1733534064285.8cd2040dc039667d68bc1a24f5f2322f. 2024-12-07T01:14:28,404 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,www,1733534064285.8cd2040dc039667d68bc1a24f5f2322f. 2024-12-07T01:14:28,404 INFO [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,rrr,1733534064285.fa4e20ed49e59ee4cf63b6d84515872f. 2024-12-07T01:14:28,404 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,www,1733534064285.8cd2040dc039667d68bc1a24f5f2322f. after waiting 0 ms 2024-12-07T01:14:28,404 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,rrr,1733534064285.fa4e20ed49e59ee4cf63b6d84515872f. 2024-12-07T01:14:28,404 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,www,1733534064285.8cd2040dc039667d68bc1a24f5f2322f. 2024-12-07T01:14:28,405 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,rrr,1733534064285.fa4e20ed49e59ee4cf63b6d84515872f. after waiting 0 ms 2024-12-07T01:14:28,405 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,rrr,1733534064285.fa4e20ed49e59ee4cf63b6d84515872f. 2024-12-07T01:14:28,409 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/default/test/8cd2040dc039667d68bc1a24f5f2322f/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-07T01:14:28,409 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/default/test/fa4e20ed49e59ee4cf63b6d84515872f/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-07T01:14:28,409 INFO [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,www,1733534064285.8cd2040dc039667d68bc1a24f5f2322f. 2024-12-07T01:14:28,410 INFO [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,rrr,1733534064285.fa4e20ed49e59ee4cf63b6d84515872f. 2024-12-07T01:14:28,410 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 8cd2040dc039667d68bc1a24f5f2322f: Waiting for close lock at 1733534068404Running coprocessor pre-close hooks at 1733534068404Disabling compacts and flushes for region at 1733534068404Disabling writes for close at 1733534068404Writing region close event to WAL at 1733534068405 (+1 ms)Running coprocessor post-close hooks at 1733534068409 (+4 ms)Closed at 1733534068409 2024-12-07T01:14:28,410 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for fa4e20ed49e59ee4cf63b6d84515872f: Waiting for close lock at 1733534068404Running coprocessor pre-close hooks at 1733534068404Disabling compacts and flushes for region at 1733534068404Disabling writes for close at 1733534068405 (+1 ms)Writing region close event to WAL at 1733534068405Running coprocessor post-close hooks at 1733534068409 (+4 ms)Closed at 1733534068409 2024-12-07T01:14:28,410 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,rrr,1733534064285.fa4e20ed49e59ee4cf63b6d84515872f. 2024-12-07T01:14:28,410 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,www,1733534064285.8cd2040dc039667d68bc1a24f5f2322f. 2024-12-07T01:14:28,410 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 5d3306d206c48a1c7060e49d4fe00ad5, disabling compactions & flushes 2024-12-07T01:14:28,410 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing b59f40eb016096571b13ae445f262449, disabling compactions & flushes 2024-12-07T01:14:28,410 INFO [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,iii,1733534064285.5d3306d206c48a1c7060e49d4fe00ad5. 2024-12-07T01:14:28,410 INFO [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,yyy,1733534064285.b59f40eb016096571b13ae445f262449. 2024-12-07T01:14:28,410 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,iii,1733534064285.5d3306d206c48a1c7060e49d4fe00ad5. 2024-12-07T01:14:28,410 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,yyy,1733534064285.b59f40eb016096571b13ae445f262449. 2024-12-07T01:14:28,410 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,iii,1733534064285.5d3306d206c48a1c7060e49d4fe00ad5. after waiting 0 ms 2024-12-07T01:14:28,410 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,yyy,1733534064285.b59f40eb016096571b13ae445f262449. after waiting 0 ms 2024-12-07T01:14:28,410 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,iii,1733534064285.5d3306d206c48a1c7060e49d4fe00ad5. 2024-12-07T01:14:28,410 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,yyy,1733534064285.b59f40eb016096571b13ae445f262449. 2024-12-07T01:14:28,414 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/default/test/b59f40eb016096571b13ae445f262449/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-07T01:14:28,414 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/default/test/5d3306d206c48a1c7060e49d4fe00ad5/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-07T01:14:28,414 INFO [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,yyy,1733534064285.b59f40eb016096571b13ae445f262449. 2024-12-07T01:14:28,414 INFO [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,iii,1733534064285.5d3306d206c48a1c7060e49d4fe00ad5. 2024-12-07T01:14:28,414 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for b59f40eb016096571b13ae445f262449: Waiting for close lock at 1733534068410Running coprocessor pre-close hooks at 1733534068410Disabling compacts and flushes for region at 1733534068410Disabling writes for close at 1733534068410Writing region close event to WAL at 1733534068410Running coprocessor post-close hooks at 1733534068414 (+4 ms)Closed at 1733534068414 2024-12-07T01:14:28,414 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 5d3306d206c48a1c7060e49d4fe00ad5: Waiting for close lock at 1733534068410Running coprocessor pre-close hooks at 1733534068410Disabling compacts and flushes for region at 1733534068410Disabling writes for close at 1733534068410Writing region close event to WAL at 1733534068410Running coprocessor post-close hooks at 1733534068414 (+4 ms)Closed at 1733534068414 2024-12-07T01:14:28,414 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,iii,1733534064285.5d3306d206c48a1c7060e49d4fe00ad5. 2024-12-07T01:14:28,414 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,yyy,1733534064285.b59f40eb016096571b13ae445f262449. 2024-12-07T01:14:28,415 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 37d131a6ac713be02dc73a29079210f5, disabling compactions & flushes 2024-12-07T01:14:28,415 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 0980fd6969c12f09b843d772c626dfa5, disabling compactions & flushes 2024-12-07T01:14:28,415 INFO [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,fff,1733534064285.37d131a6ac713be02dc73a29079210f5. 2024-12-07T01:14:28,415 INFO [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,bbb,1733534064285.0980fd6969c12f09b843d772c626dfa5. 2024-12-07T01:14:28,415 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,fff,1733534064285.37d131a6ac713be02dc73a29079210f5. 2024-12-07T01:14:28,415 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,bbb,1733534064285.0980fd6969c12f09b843d772c626dfa5. 2024-12-07T01:14:28,415 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,fff,1733534064285.37d131a6ac713be02dc73a29079210f5. after waiting 0 ms 2024-12-07T01:14:28,415 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,bbb,1733534064285.0980fd6969c12f09b843d772c626dfa5. after waiting 0 ms 2024-12-07T01:14:28,415 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,fff,1733534064285.37d131a6ac713be02dc73a29079210f5. 2024-12-07T01:14:28,415 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,bbb,1733534064285.0980fd6969c12f09b843d772c626dfa5. 2024-12-07T01:14:28,418 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/default/test/37d131a6ac713be02dc73a29079210f5/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-07T01:14:28,418 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/default/test/0980fd6969c12f09b843d772c626dfa5/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-07T01:14:28,419 INFO [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,fff,1733534064285.37d131a6ac713be02dc73a29079210f5. 2024-12-07T01:14:28,419 INFO [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,bbb,1733534064285.0980fd6969c12f09b843d772c626dfa5. 2024-12-07T01:14:28,419 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 37d131a6ac713be02dc73a29079210f5: Waiting for close lock at 1733534068414Running coprocessor pre-close hooks at 1733534068414Disabling compacts and flushes for region at 1733534068414Disabling writes for close at 1733534068415 (+1 ms)Writing region close event to WAL at 1733534068415Running coprocessor post-close hooks at 1733534068419 (+4 ms)Closed at 1733534068419 2024-12-07T01:14:28,419 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 0980fd6969c12f09b843d772c626dfa5: Waiting for close lock at 1733534068414Running coprocessor pre-close hooks at 1733534068414Disabling compacts and flushes for region at 1733534068415 (+1 ms)Disabling writes for close at 1733534068415Writing region close event to WAL at 1733534068415Running coprocessor post-close hooks at 1733534068419 (+4 ms)Closed at 1733534068419 2024-12-07T01:14:28,419 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,fff,1733534064285.37d131a6ac713be02dc73a29079210f5. 2024-12-07T01:14:28,419 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,bbb,1733534064285.0980fd6969c12f09b843d772c626dfa5. 2024-12-07T01:14:28,419 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing aa3dca7b6c799eb77565fb0e20ef1c15, disabling compactions & flushes 2024-12-07T01:14:28,419 INFO [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,jjj,1733534064285.aa3dca7b6c799eb77565fb0e20ef1c15. 2024-12-07T01:14:28,419 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,jjj,1733534064285.aa3dca7b6c799eb77565fb0e20ef1c15. 2024-12-07T01:14:28,419 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,jjj,1733534064285.aa3dca7b6c799eb77565fb0e20ef1c15. after waiting 0 ms 2024-12-07T01:14:28,419 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,jjj,1733534064285.aa3dca7b6c799eb77565fb0e20ef1c15. 2024-12-07T01:14:28,423 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/data/default/test/aa3dca7b6c799eb77565fb0e20ef1c15/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-07T01:14:28,424 INFO [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,jjj,1733534064285.aa3dca7b6c799eb77565fb0e20ef1c15. 2024-12-07T01:14:28,424 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for aa3dca7b6c799eb77565fb0e20ef1c15: Waiting for close lock at 1733534068419Running coprocessor pre-close hooks at 1733534068419Disabling compacts and flushes for region at 1733534068419Disabling writes for close at 1733534068419Writing region close event to WAL at 1733534068420 (+1 ms)Running coprocessor post-close hooks at 1733534068424 (+4 ms)Closed at 1733534068424 2024-12-07T01:14:28,424 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,jjj,1733534064285.aa3dca7b6c799eb77565fb0e20ef1c15. 2024-12-07T01:14:28,547 INFO [RS:0;ec1863dc21e5:42915 {}] regionserver.HRegionServer(976): stopping server ec1863dc21e5,42915,1733534061580; all regions closed. 2024-12-07T01:14:28,547 INFO [RS:2;ec1863dc21e5:40693 {}] regionserver.HRegionServer(976): stopping server ec1863dc21e5,40693,1733534061744; all regions closed. 2024-12-07T01:14:28,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37235 is added to blk_1073741865_1041 (size=4204) 2024-12-07T01:14:28,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741835_1011 (size=7074) 2024-12-07T01:14:28,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741865_1041 (size=4204) 2024-12-07T01:14:28,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43991 is added to blk_1073741865_1041 (size=4204) 2024-12-07T01:14:28,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37235 is added to blk_1073741835_1011 (size=7074) 2024-12-07T01:14:28,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43991 is added to blk_1073741835_1011 (size=7074) 2024-12-07T01:14:28,553 DEBUG [RS:2;ec1863dc21e5:40693 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/oldWALs 2024-12-07T01:14:28,554 INFO [RS:2;ec1863dc21e5:40693 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL ec1863dc21e5%2C40693%2C1733534061744.meta:.meta(num 1733534066417) 2024-12-07T01:14:28,554 DEBUG [RS:0;ec1863dc21e5:42915 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/oldWALs 2024-12-07T01:14:28,554 INFO [RS:0;ec1863dc21e5:42915 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL ec1863dc21e5%2C42915%2C1733534061580:(num 1733534063448) 2024-12-07T01:14:28,554 DEBUG [RS:0;ec1863dc21e5:42915 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T01:14:28,554 INFO [RS:0;ec1863dc21e5:42915 {}] regionserver.LeaseManager(133): Closed leases 2024-12-07T01:14:28,554 INFO [RS:0;ec1863dc21e5:42915 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-07T01:14:28,555 INFO [RS:0;ec1863dc21e5:42915 {}] hbase.ChoreService(370): Chore service for: regionserver/ec1863dc21e5:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-07T01:14:28,555 INFO [RS:0;ec1863dc21e5:42915 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-07T01:14:28,555 INFO [RS:0;ec1863dc21e5:42915 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-07T01:14:28,555 INFO [regionserver/ec1863dc21e5:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-07T01:14:28,555 INFO [RS:0;ec1863dc21e5:42915 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-07T01:14:28,555 INFO [RS:0;ec1863dc21e5:42915 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-07T01:14:28,555 INFO [RS:0;ec1863dc21e5:42915 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:42915 2024-12-07T01:14:28,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741833_1009 (size=7064) 2024-12-07T01:14:28,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37235 is added to blk_1073741833_1009 (size=7064) 2024-12-07T01:14:28,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43991 is added to blk_1073741833_1009 (size=7064) 2024-12-07T01:14:28,559 DEBUG [RS:2;ec1863dc21e5:40693 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/oldWALs 2024-12-07T01:14:28,559 INFO [RS:2;ec1863dc21e5:40693 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL ec1863dc21e5%2C40693%2C1733534061744:(num 1733534063446) 2024-12-07T01:14:28,559 DEBUG [RS:2;ec1863dc21e5:40693 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T01:14:28,559 INFO [RS:2;ec1863dc21e5:40693 {}] regionserver.LeaseManager(133): Closed leases 2024-12-07T01:14:28,559 INFO [RS:2;ec1863dc21e5:40693 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-07T01:14:28,559 INFO [RS:2;ec1863dc21e5:40693 {}] hbase.ChoreService(370): Chore service for: regionserver/ec1863dc21e5:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-07T01:14:28,559 INFO [RS:2;ec1863dc21e5:40693 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-07T01:14:28,560 INFO [regionserver/ec1863dc21e5:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-07T01:14:28,560 INFO [RS:2;ec1863dc21e5:40693 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:40693 2024-12-07T01:14:28,564 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46141-0x101ad5a899b0000, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-07T01:14:28,564 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42915-0x101ad5a899b0001, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/ec1863dc21e5,42915,1733534061580 2024-12-07T01:14:28,564 INFO [RS:0;ec1863dc21e5:42915 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-07T01:14:28,575 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40693-0x101ad5a899b0003, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/ec1863dc21e5,40693,1733534061744 2024-12-07T01:14:28,575 INFO [RS:2;ec1863dc21e5:40693 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-07T01:14:28,585 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [ec1863dc21e5,40693,1733534061744] 2024-12-07T01:14:28,606 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/ec1863dc21e5,40693,1733534061744 already deleted, retry=false 2024-12-07T01:14:28,606 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; ec1863dc21e5,40693,1733534061744 expired; onlineServers=1 2024-12-07T01:14:28,606 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [ec1863dc21e5,42915,1733534061580] 2024-12-07T01:14:28,617 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/ec1863dc21e5,42915,1733534061580 already deleted, retry=false 2024-12-07T01:14:28,617 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; ec1863dc21e5,42915,1733534061580 expired; onlineServers=0 2024-12-07T01:14:28,617 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'ec1863dc21e5,46141,1733534060840' ***** 2024-12-07T01:14:28,617 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-07T01:14:28,617 INFO [M:0;ec1863dc21e5:46141 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-07T01:14:28,617 INFO [M:0;ec1863dc21e5:46141 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-07T01:14:28,617 DEBUG [M:0;ec1863dc21e5:46141 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-07T01:14:28,617 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-07T01:14:28,617 DEBUG [M:0;ec1863dc21e5:46141 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-07T01:14:28,617 DEBUG [master/ec1863dc21e5:0:becomeActiveMaster-HFileCleaner.small.0-1733534063110 {}] cleaner.HFileCleaner(306): Exit Thread[master/ec1863dc21e5:0:becomeActiveMaster-HFileCleaner.small.0-1733534063110,5,FailOnTimeoutGroup] 2024-12-07T01:14:28,617 DEBUG [master/ec1863dc21e5:0:becomeActiveMaster-HFileCleaner.large.0-1733534063109 {}] cleaner.HFileCleaner(306): Exit Thread[master/ec1863dc21e5:0:becomeActiveMaster-HFileCleaner.large.0-1733534063109,5,FailOnTimeoutGroup] 2024-12-07T01:14:28,618 INFO [M:0;ec1863dc21e5:46141 {}] hbase.ChoreService(370): Chore service for: master/ec1863dc21e5:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-07T01:14:28,618 INFO [M:0;ec1863dc21e5:46141 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-07T01:14:28,618 DEBUG [M:0;ec1863dc21e5:46141 {}] master.HMaster(1795): Stopping service threads 2024-12-07T01:14:28,618 INFO [M:0;ec1863dc21e5:46141 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-07T01:14:28,618 INFO [M:0;ec1863dc21e5:46141 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-07T01:14:28,618 INFO [M:0;ec1863dc21e5:46141 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-07T01:14:28,618 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-07T01:14:28,627 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46141-0x101ad5a899b0000, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-07T01:14:28,627 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46141-0x101ad5a899b0000, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T01:14:28,627 DEBUG [M:0;ec1863dc21e5:46141 {}] zookeeper.ZKUtil(347): master:46141-0x101ad5a899b0000, quorum=127.0.0.1:61706, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-07T01:14:28,627 WARN [M:0;ec1863dc21e5:46141 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-07T01:14:28,628 INFO [M:0;ec1863dc21e5:46141 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/.lastflushedseqids 2024-12-07T01:14:28,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37235 is added to blk_1073741870_1046 (size=1) 2024-12-07T01:14:28,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741870_1046 (size=1) 2024-12-07T01:14:28,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43991 is added to blk_1073741870_1046 (size=1) 2024-12-07T01:14:28,638 INFO [M:0;ec1863dc21e5:46141 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-07T01:14:28,638 INFO [M:0;ec1863dc21e5:46141 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-07T01:14:28,638 DEBUG [M:0;ec1863dc21e5:46141 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-07T01:14:28,638 INFO [M:0;ec1863dc21e5:46141 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-07T01:14:28,639 DEBUG [M:0;ec1863dc21e5:46141 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-07T01:14:28,639 DEBUG [M:0;ec1863dc21e5:46141 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-07T01:14:28,639 DEBUG [M:0;ec1863dc21e5:46141 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-07T01:14:28,639 INFO [M:0;ec1863dc21e5:46141 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=250.11 KB heapSize=303.94 KB 2024-12-07T01:14:28,653 DEBUG [M:0;ec1863dc21e5:46141 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/16c70a757c0b461f862d7504b8e32e71 is 82, key is hbase:meta,,1/info:regioninfo/1733534066682/Put/seqid=0 2024-12-07T01:14:28,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37235 is added to blk_1073741871_1047 (size=6071) 2024-12-07T01:14:28,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741871_1047 (size=6071) 2024-12-07T01:14:28,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43991 is added to blk_1073741871_1047 (size=6071) 2024-12-07T01:14:28,660 INFO [M:0;ec1863dc21e5:46141 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=575 (bloomFilter=true), to=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/16c70a757c0b461f862d7504b8e32e71 2024-12-07T01:14:28,680 DEBUG [M:0;ec1863dc21e5:46141 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c174a2e4b5f4406cb1f1671642ccb040 is 2990, key is \x00\x00\x00\x00\x00\x00\x009/proc:d/1733534068314/Put/seqid=0 2024-12-07T01:14:28,685 INFO [RS:0;ec1863dc21e5:42915 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-07T01:14:28,685 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42915-0x101ad5a899b0001, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-07T01:14:28,685 INFO [RS:0;ec1863dc21e5:42915 {}] regionserver.HRegionServer(1031): Exiting; stopping=ec1863dc21e5,42915,1733534061580; zookeeper connection closed. 2024-12-07T01:14:28,685 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42915-0x101ad5a899b0001, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-07T01:14:28,686 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@684bced5 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@684bced5 2024-12-07T01:14:28,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37235 is added to blk_1073741872_1048 (size=12801) 2024-12-07T01:14:28,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43991 is added to blk_1073741872_1048 (size=12801) 2024-12-07T01:14:28,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741872_1048 (size=12801) 2024-12-07T01:14:28,688 INFO [M:0;ec1863dc21e5:46141 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=248.83 KB at sequenceid=575 (bloomFilter=true), to=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c174a2e4b5f4406cb1f1671642ccb040 2024-12-07T01:14:28,696 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40693-0x101ad5a899b0003, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-07T01:14:28,696 INFO [RS:2;ec1863dc21e5:40693 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-07T01:14:28,696 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40693-0x101ad5a899b0003, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-07T01:14:28,696 INFO [RS:2;ec1863dc21e5:40693 {}] regionserver.HRegionServer(1031): Exiting; stopping=ec1863dc21e5,40693,1733534061744; zookeeper connection closed. 2024-12-07T01:14:28,696 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@3a359d0b {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@3a359d0b 2024-12-07T01:14:28,697 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-12-07T01:14:28,707 DEBUG [M:0;ec1863dc21e5:46141 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/5ffb6ca4924345e19ffde71c1abbea20 is 69, key is ec1863dc21e5,40693,1733534061744/rs:state/1733534063130/Put/seqid=0 2024-12-07T01:14:28,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37235 is added to blk_1073741873_1049 (size=5442) 2024-12-07T01:14:28,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43991 is added to blk_1073741873_1049 (size=5442) 2024-12-07T01:14:28,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741873_1049 (size=5442) 2024-12-07T01:14:28,714 INFO [M:0;ec1863dc21e5:46141 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=575 (bloomFilter=true), to=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/5ffb6ca4924345e19ffde71c1abbea20 2024-12-07T01:14:28,720 INFO [M:0;ec1863dc21e5:46141 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 5ffb6ca4924345e19ffde71c1abbea20 2024-12-07T01:14:28,733 DEBUG [M:0;ec1863dc21e5:46141 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/9621169c657d4bee96570bd763ea9fdc is 52, key is load_balancer_on/state:d/1733534065604/Put/seqid=0 2024-12-07T01:14:28,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43991 is added to blk_1073741874_1050 (size=5058) 2024-12-07T01:14:28,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37235 is added to blk_1073741874_1050 (size=5058) 2024-12-07T01:14:28,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741874_1050 (size=5058) 2024-12-07T01:14:28,740 INFO [M:0;ec1863dc21e5:46141 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=575 (bloomFilter=true), to=hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/9621169c657d4bee96570bd763ea9fdc 2024-12-07T01:14:28,746 DEBUG [M:0;ec1863dc21e5:46141 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/16c70a757c0b461f862d7504b8e32e71 as hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/16c70a757c0b461f862d7504b8e32e71 2024-12-07T01:14:28,751 INFO [M:0;ec1863dc21e5:46141 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/16c70a757c0b461f862d7504b8e32e71, entries=14, sequenceid=575, filesize=5.9 K 2024-12-07T01:14:28,753 DEBUG [M:0;ec1863dc21e5:46141 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c174a2e4b5f4406cb1f1671642ccb040 as hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/c174a2e4b5f4406cb1f1671642ccb040 2024-12-07T01:14:28,758 INFO [M:0;ec1863dc21e5:46141 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/c174a2e4b5f4406cb1f1671642ccb040, entries=79, sequenceid=575, filesize=12.5 K 2024-12-07T01:14:28,759 DEBUG [M:0;ec1863dc21e5:46141 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/5ffb6ca4924345e19ffde71c1abbea20 as hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/5ffb6ca4924345e19ffde71c1abbea20 2024-12-07T01:14:28,765 INFO [M:0;ec1863dc21e5:46141 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 5ffb6ca4924345e19ffde71c1abbea20 2024-12-07T01:14:28,766 INFO [M:0;ec1863dc21e5:46141 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/5ffb6ca4924345e19ffde71c1abbea20, entries=3, sequenceid=575, filesize=5.3 K 2024-12-07T01:14:28,767 DEBUG [M:0;ec1863dc21e5:46141 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/9621169c657d4bee96570bd763ea9fdc as hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/9621169c657d4bee96570bd763ea9fdc 2024-12-07T01:14:28,773 INFO [M:0;ec1863dc21e5:46141 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35877/user/jenkins/test-data/4b5f3771-7166-3534-e2dd-e7c6d7f09323/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/9621169c657d4bee96570bd763ea9fdc, entries=1, sequenceid=575, filesize=4.9 K 2024-12-07T01:14:28,774 INFO [M:0;ec1863dc21e5:46141 {}] regionserver.HRegion(3140): Finished flush of dataSize ~250.11 KB/256108, heapSize ~303.88 KB/311168, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 134ms, sequenceid=575, compaction requested=false 2024-12-07T01:14:28,775 INFO [M:0;ec1863dc21e5:46141 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-07T01:14:28,775 DEBUG [M:0;ec1863dc21e5:46141 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733534068638Disabling compacts and flushes for region at 1733534068638Disabling writes for close at 1733534068639 (+1 ms)Obtaining lock to block concurrent updates at 1733534068639Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733534068639Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=256108, getHeapSize=311168, getOffHeapSize=0, getCellsCount=714 at 1733534068639Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733534068640 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733534068640Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733534068653 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733534068653Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733534068665 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733534068679 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733534068679Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733534068693 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733534068707 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733534068707Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1733534068720 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1733534068733 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1733534068733Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7395cfb2: reopening flushed file at 1733534068745 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@19c9611: reopening flushed file at 1733534068752 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@22bb50b3: reopening flushed file at 1733534068758 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2698b65d: reopening flushed file at 1733534068766 (+8 ms)Finished flush of dataSize ~250.11 KB/256108, heapSize ~303.88 KB/311168, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 134ms, sequenceid=575, compaction requested=false at 1733534068774 (+8 ms)Writing region close event to WAL at 1733534068775 (+1 ms)Closed at 1733534068775 2024-12-07T01:14:28,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37235 is added to blk_1073741830_1006 (size=294492) 2024-12-07T01:14:28,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741830_1006 (size=294492) 2024-12-07T01:14:28,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43991 is added to blk_1073741830_1006 (size=294492) 2024-12-07T01:14:28,778 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-07T01:14:28,778 INFO [M:0;ec1863dc21e5:46141 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-07T01:14:28,778 INFO [M:0;ec1863dc21e5:46141 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:46141 2024-12-07T01:14:28,779 INFO [M:0;ec1863dc21e5:46141 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-07T01:14:28,891 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46141-0x101ad5a899b0000, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-07T01:14:28,891 INFO [M:0;ec1863dc21e5:46141 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-07T01:14:28,891 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46141-0x101ad5a899b0000, quorum=127.0.0.1:61706, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-07T01:14:28,893 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-07T01:14:28,929 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3c1f701{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-07T01:14:28,935 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3d599bb4{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-07T01:14:28,935 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-07T01:14:28,935 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@417efacb{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-07T01:14:28,936 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@290e2eec{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4e4c318f-1ef6-1cff-e733-20a2b141900e/hadoop.log.dir/,STOPPED} 2024-12-07T01:14:28,938 WARN [BP-1976987596-172.17.0.3-1733534056320 heartbeating to localhost/127.0.0.1:35877 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-07T01:14:28,938 WARN [BP-1976987596-172.17.0.3-1733534056320 heartbeating to localhost/127.0.0.1:35877 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1976987596-172.17.0.3-1733534056320 (Datanode Uuid 41e996fa-a8fa-4e8a-9869-a72e6d9121e5) service to localhost/127.0.0.1:35877 2024-12-07T01:14:28,940 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4e4c318f-1ef6-1cff-e733-20a2b141900e/cluster_25b350fe-02a5-c1aa-017d-c8a3f951e395/data/data5/current/BP-1976987596-172.17.0.3-1733534056320 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-07T01:14:28,940 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4e4c318f-1ef6-1cff-e733-20a2b141900e/cluster_25b350fe-02a5-c1aa-017d-c8a3f951e395/data/data6/current/BP-1976987596-172.17.0.3-1733534056320 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-07T01:14:28,940 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-07T01:14:28,942 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@34fcb585{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-07T01:14:28,942 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@389bb6f7{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-07T01:14:28,943 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-07T01:14:28,943 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1b0a32ba{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-07T01:14:28,943 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@29652cf4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4e4c318f-1ef6-1cff-e733-20a2b141900e/hadoop.log.dir/,STOPPED} 2024-12-07T01:14:28,944 WARN [BP-1976987596-172.17.0.3-1733534056320 heartbeating to localhost/127.0.0.1:35877 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-07T01:14:28,944 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-07T01:14:28,945 WARN [BP-1976987596-172.17.0.3-1733534056320 heartbeating to localhost/127.0.0.1:35877 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1976987596-172.17.0.3-1733534056320 (Datanode Uuid 8b81b8a3-bf4e-48e3-8ddb-69e75519470d) service to localhost/127.0.0.1:35877 2024-12-07T01:14:28,945 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-07T01:14:28,945 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4e4c318f-1ef6-1cff-e733-20a2b141900e/cluster_25b350fe-02a5-c1aa-017d-c8a3f951e395/data/data3/current/BP-1976987596-172.17.0.3-1733534056320 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-07T01:14:28,945 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4e4c318f-1ef6-1cff-e733-20a2b141900e/cluster_25b350fe-02a5-c1aa-017d-c8a3f951e395/data/data4/current/BP-1976987596-172.17.0.3-1733534056320 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-07T01:14:28,946 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-07T01:14:28,947 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@79b30fb0{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-07T01:14:28,948 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@548cd725{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-07T01:14:28,948 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-07T01:14:28,948 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@474d518f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-07T01:14:28,948 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3a300759{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4e4c318f-1ef6-1cff-e733-20a2b141900e/hadoop.log.dir/,STOPPED} 2024-12-07T01:14:28,950 WARN [BP-1976987596-172.17.0.3-1733534056320 heartbeating to localhost/127.0.0.1:35877 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-07T01:14:28,950 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-07T01:14:28,950 WARN [BP-1976987596-172.17.0.3-1733534056320 heartbeating to localhost/127.0.0.1:35877 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1976987596-172.17.0.3-1733534056320 (Datanode Uuid 48fe2fef-5e13-4e48-b03f-fb4a2b850877) service to localhost/127.0.0.1:35877 2024-12-07T01:14:28,950 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-07T01:14:28,950 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4e4c318f-1ef6-1cff-e733-20a2b141900e/cluster_25b350fe-02a5-c1aa-017d-c8a3f951e395/data/data1/current/BP-1976987596-172.17.0.3-1733534056320 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-07T01:14:28,950 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4e4c318f-1ef6-1cff-e733-20a2b141900e/cluster_25b350fe-02a5-c1aa-017d-c8a3f951e395/data/data2/current/BP-1976987596-172.17.0.3-1733534056320 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-07T01:14:28,951 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-07T01:14:28,958 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@64418798{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-07T01:14:28,959 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@11744524{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-07T01:14:28,959 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-07T01:14:28,959 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@595320c8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-07T01:14:28,959 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3912242{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4e4c318f-1ef6-1cff-e733-20a2b141900e/hadoop.log.dir/,STOPPED} 2024-12-07T01:14:28,990 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-07T01:14:29,036 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down