2024-11-20 03:00:42,990 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 2024-11-20 03:00:43,003 main DEBUG Took 0.010949 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-20 03:00:43,004 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-20 03:00:43,004 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-20 03:00:43,005 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-20 03:00:43,006 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 03:00:43,014 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-20 03:00:43,030 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 03:00:43,032 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 03:00:43,033 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 03:00:43,033 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 03:00:43,034 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 03:00:43,035 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 03:00:43,036 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 03:00:43,037 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 03:00:43,037 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 03:00:43,038 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 03:00:43,039 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 03:00:43,039 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 03:00:43,040 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 03:00:43,041 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 03:00:43,041 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 03:00:43,042 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 03:00:43,043 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 03:00:43,043 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 03:00:43,044 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 03:00:43,044 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 03:00:43,045 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 03:00:43,045 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 03:00:43,046 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 03:00:43,047 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 03:00:43,047 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 03:00:43,048 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-20 03:00:43,050 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 03:00:43,052 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-20 03:00:43,054 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-20 03:00:43,055 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-20 03:00:43,057 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-20 03:00:43,057 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-20 03:00:43,068 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-20 03:00:43,072 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-20 03:00:43,074 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-20 03:00:43,074 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-20 03:00:43,075 main DEBUG createAppenders(={Console}) 2024-11-20 03:00:43,076 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 initialized 2024-11-20 03:00:43,076 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 2024-11-20 03:00:43,077 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 OK. 2024-11-20 03:00:43,077 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-20 03:00:43,078 main DEBUG OutputStream closed 2024-11-20 03:00:43,078 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-20 03:00:43,078 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-20 03:00:43,079 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@c88a337 OK 2024-11-20 03:00:43,156 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-20 03:00:43,158 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-20 03:00:43,160 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-20 03:00:43,161 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-20 03:00:43,161 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-20 03:00:43,162 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-20 03:00:43,162 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-20 03:00:43,163 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-20 03:00:43,163 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-20 03:00:43,163 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-20 03:00:43,164 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-20 03:00:43,164 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-20 03:00:43,165 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-20 03:00:43,165 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-20 03:00:43,166 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-20 03:00:43,166 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-20 03:00:43,166 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-20 03:00:43,167 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-20 03:00:43,169 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-20 03:00:43,169 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@13330ac6) with optional ClassLoader: null 2024-11-20 03:00:43,170 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-20 03:00:43,171 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@13330ac6] started OK. 2024-11-20T03:00:43,183 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.master.assignment.TestRollbackSCP timeout: 13 mins 2024-11-20 03:00:43,186 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-20 03:00:43,186 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-20T03:00:43,448 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cc48e81-f8f9-9439-aadc-f08e2410e401 2024-11-20T03:00:43,473 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=class org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$HMasterForTest, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-20T03:00:43,494 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cc48e81-f8f9-9439-aadc-f08e2410e401/cluster_a1accffb-3df0-3830-8589-ade335756e45, deleteOnExit=true 2024-11-20T03:00:43,494 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-20T03:00:43,496 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cc48e81-f8f9-9439-aadc-f08e2410e401/test.cache.data in system properties and HBase conf 2024-11-20T03:00:43,497 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cc48e81-f8f9-9439-aadc-f08e2410e401/hadoop.tmp.dir in system properties and HBase conf 2024-11-20T03:00:43,498 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cc48e81-f8f9-9439-aadc-f08e2410e401/hadoop.log.dir in system properties and HBase conf 2024-11-20T03:00:43,499 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cc48e81-f8f9-9439-aadc-f08e2410e401/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-20T03:00:43,500 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cc48e81-f8f9-9439-aadc-f08e2410e401/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-20T03:00:43,500 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-20T03:00:43,614 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-20T03:00:43,735 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T03:00:43,740 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cc48e81-f8f9-9439-aadc-f08e2410e401/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-20T03:00:43,741 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cc48e81-f8f9-9439-aadc-f08e2410e401/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-20T03:00:43,742 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cc48e81-f8f9-9439-aadc-f08e2410e401/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-20T03:00:43,743 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cc48e81-f8f9-9439-aadc-f08e2410e401/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-20T03:00:43,743 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cc48e81-f8f9-9439-aadc-f08e2410e401/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-20T03:00:43,744 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cc48e81-f8f9-9439-aadc-f08e2410e401/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-20T03:00:43,745 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cc48e81-f8f9-9439-aadc-f08e2410e401/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-20T03:00:43,746 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cc48e81-f8f9-9439-aadc-f08e2410e401/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-20T03:00:43,747 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cc48e81-f8f9-9439-aadc-f08e2410e401/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-20T03:00:43,747 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cc48e81-f8f9-9439-aadc-f08e2410e401/nfs.dump.dir in system properties and HBase conf 2024-11-20T03:00:43,748 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cc48e81-f8f9-9439-aadc-f08e2410e401/java.io.tmpdir in system properties and HBase conf 2024-11-20T03:00:43,748 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cc48e81-f8f9-9439-aadc-f08e2410e401/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-20T03:00:43,749 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cc48e81-f8f9-9439-aadc-f08e2410e401/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-20T03:00:43,750 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cc48e81-f8f9-9439-aadc-f08e2410e401/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-20T03:00:44,935 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-20T03:00:45,053 INFO [Time-limited test {}] log.Log(170): Logging initialized @2950ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-20T03:00:45,160 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-20T03:00:45,248 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-20T03:00:45,278 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-20T03:00:45,279 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-20T03:00:45,280 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-20T03:00:45,297 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-20T03:00:45,300 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@265b24f1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cc48e81-f8f9-9439-aadc-f08e2410e401/hadoop.log.dir/,AVAILABLE} 2024-11-20T03:00:45,301 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@76beb053{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-20T03:00:45,477 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@376fa8f3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cc48e81-f8f9-9439-aadc-f08e2410e401/java.io.tmpdir/jetty-localhost-45423-hadoop-hdfs-3_4_1-tests_jar-_-any-4947981665046262854/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-20T03:00:45,483 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@672b88ea{HTTP/1.1, (http/1.1)}{localhost:45423} 2024-11-20T03:00:45,483 INFO [Time-limited test {}] server.Server(415): Started @3382ms 2024-11-20T03:00:46,097 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-20T03:00:46,109 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-20T03:00:46,111 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-20T03:00:46,111 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-20T03:00:46,111 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-20T03:00:46,112 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@739a104e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cc48e81-f8f9-9439-aadc-f08e2410e401/hadoop.log.dir/,AVAILABLE} 2024-11-20T03:00:46,113 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1c5d2af1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-20T03:00:46,224 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2226d6a6{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cc48e81-f8f9-9439-aadc-f08e2410e401/java.io.tmpdir/jetty-localhost-34329-hadoop-hdfs-3_4_1-tests_jar-_-any-15350710704491122310/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-20T03:00:46,224 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@1da1f773{HTTP/1.1, (http/1.1)}{localhost:34329} 2024-11-20T03:00:46,225 INFO [Time-limited test {}] server.Server(415): Started @4123ms 2024-11-20T03:00:46,271 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-20T03:00:46,404 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-20T03:00:46,411 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-20T03:00:46,422 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-20T03:00:46,422 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-20T03:00:46,423 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-20T03:00:46,426 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@34a3e1ff{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cc48e81-f8f9-9439-aadc-f08e2410e401/hadoop.log.dir/,AVAILABLE} 2024-11-20T03:00:46,427 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@12f16529{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-20T03:00:46,541 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7cfccf27{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cc48e81-f8f9-9439-aadc-f08e2410e401/java.io.tmpdir/jetty-localhost-37745-hadoop-hdfs-3_4_1-tests_jar-_-any-6433295416240197/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-20T03:00:46,542 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6c41bbb7{HTTP/1.1, (http/1.1)}{localhost:37745} 2024-11-20T03:00:46,542 INFO [Time-limited test {}] server.Server(415): Started @4441ms 2024-11-20T03:00:46,544 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-20T03:00:46,594 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-20T03:00:46,601 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-20T03:00:46,607 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-20T03:00:46,608 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-20T03:00:46,608 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-20T03:00:46,611 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@11ebaa12{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cc48e81-f8f9-9439-aadc-f08e2410e401/hadoop.log.dir/,AVAILABLE} 2024-11-20T03:00:46,612 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@52cf89e6{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-20T03:00:46,721 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@67ddfd25{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cc48e81-f8f9-9439-aadc-f08e2410e401/java.io.tmpdir/jetty-localhost-38181-hadoop-hdfs-3_4_1-tests_jar-_-any-14305072571779008641/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-20T03:00:46,722 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@30e38263{HTTP/1.1, (http/1.1)}{localhost:38181} 2024-11-20T03:00:46,722 INFO [Time-limited test {}] server.Server(415): Started @4620ms 2024-11-20T03:00:46,725 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-20T03:00:47,707 WARN [Thread-125 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cc48e81-f8f9-9439-aadc-f08e2410e401/cluster_a1accffb-3df0-3830-8589-ade335756e45/data/data4/current/BP-1182860866-172.17.0.2-1732071644397/current, will proceed with Du for space computation calculation, 2024-11-20T03:00:47,707 WARN [Thread-124 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cc48e81-f8f9-9439-aadc-f08e2410e401/cluster_a1accffb-3df0-3830-8589-ade335756e45/data/data3/current/BP-1182860866-172.17.0.2-1732071644397/current, will proceed with Du for space computation calculation, 2024-11-20T03:00:47,707 WARN [Thread-123 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cc48e81-f8f9-9439-aadc-f08e2410e401/cluster_a1accffb-3df0-3830-8589-ade335756e45/data/data2/current/BP-1182860866-172.17.0.2-1732071644397/current, will proceed with Du for space computation calculation, 2024-11-20T03:00:47,707 WARN [Thread-122 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cc48e81-f8f9-9439-aadc-f08e2410e401/cluster_a1accffb-3df0-3830-8589-ade335756e45/data/data1/current/BP-1182860866-172.17.0.2-1732071644397/current, will proceed with Du for space computation calculation, 2024-11-20T03:00:47,732 WARN [Thread-132 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cc48e81-f8f9-9439-aadc-f08e2410e401/cluster_a1accffb-3df0-3830-8589-ade335756e45/data/data5/current/BP-1182860866-172.17.0.2-1732071644397/current, will proceed with Du for space computation calculation, 2024-11-20T03:00:47,732 WARN [Thread-133 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cc48e81-f8f9-9439-aadc-f08e2410e401/cluster_a1accffb-3df0-3830-8589-ade335756e45/data/data6/current/BP-1182860866-172.17.0.2-1732071644397/current, will proceed with Du for space computation calculation, 2024-11-20T03:00:47,754 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-20T03:00:47,755 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-20T03:00:47,766 WARN [Thread-103 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-20T03:00:47,810 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1489bec3c68db29e with lease ID 0xb914263cf127d1f0: Processing first storage report for DS-26e5314a-294f-49ed-8916-15422bf99b4b from datanode DatanodeRegistration(127.0.0.1:42923, datanodeUuid=500d6ffd-8429-4309-bae3-992a8c7dc324, infoPort=42603, infoSecurePort=0, ipcPort=36091, storageInfo=lv=-57;cid=testClusterID;nsid=1641879641;c=1732071644397) 2024-11-20T03:00:47,812 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1489bec3c68db29e with lease ID 0xb914263cf127d1f0: from storage DS-26e5314a-294f-49ed-8916-15422bf99b4b node DatanodeRegistration(127.0.0.1:42923, datanodeUuid=500d6ffd-8429-4309-bae3-992a8c7dc324, infoPort=42603, infoSecurePort=0, ipcPort=36091, storageInfo=lv=-57;cid=testClusterID;nsid=1641879641;c=1732071644397), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-20T03:00:47,812 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xfa7a41343d3a0c7c with lease ID 0xb914263cf127d1ee: Processing first storage report for DS-eff30b0a-4ee9-45c4-bd81-37560890eea3 from datanode DatanodeRegistration(127.0.0.1:33951, datanodeUuid=5735bcd3-4753-44a1-aeed-94e7ab4d5722, infoPort=43923, infoSecurePort=0, ipcPort=44853, storageInfo=lv=-57;cid=testClusterID;nsid=1641879641;c=1732071644397) 2024-11-20T03:00:47,813 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xfa7a41343d3a0c7c with lease ID 0xb914263cf127d1ee: from storage DS-eff30b0a-4ee9-45c4-bd81-37560890eea3 node DatanodeRegistration(127.0.0.1:33951, datanodeUuid=5735bcd3-4753-44a1-aeed-94e7ab4d5722, infoPort=43923, infoSecurePort=0, ipcPort=44853, storageInfo=lv=-57;cid=testClusterID;nsid=1641879641;c=1732071644397), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-20T03:00:47,813 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x218f085f86a807e7 with lease ID 0xb914263cf127d1ef: Processing first storage report for DS-afc4754b-ca55-4ef6-8e6d-da58b90deaa8 from datanode DatanodeRegistration(127.0.0.1:38873, datanodeUuid=e8cd692b-1714-45ca-b85e-b6498b320236, infoPort=38101, infoSecurePort=0, ipcPort=37647, storageInfo=lv=-57;cid=testClusterID;nsid=1641879641;c=1732071644397) 2024-11-20T03:00:47,813 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x218f085f86a807e7 with lease ID 0xb914263cf127d1ef: from storage DS-afc4754b-ca55-4ef6-8e6d-da58b90deaa8 node DatanodeRegistration(127.0.0.1:38873, datanodeUuid=e8cd692b-1714-45ca-b85e-b6498b320236, infoPort=38101, infoSecurePort=0, ipcPort=37647, storageInfo=lv=-57;cid=testClusterID;nsid=1641879641;c=1732071644397), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-20T03:00:47,813 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1489bec3c68db29e with lease ID 0xb914263cf127d1f0: Processing first storage report for DS-70fde8e0-d9fd-4b65-b255-2f985ff3f05e from datanode DatanodeRegistration(127.0.0.1:42923, datanodeUuid=500d6ffd-8429-4309-bae3-992a8c7dc324, infoPort=42603, infoSecurePort=0, ipcPort=36091, storageInfo=lv=-57;cid=testClusterID;nsid=1641879641;c=1732071644397) 2024-11-20T03:00:47,814 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1489bec3c68db29e with lease ID 0xb914263cf127d1f0: from storage DS-70fde8e0-d9fd-4b65-b255-2f985ff3f05e node DatanodeRegistration(127.0.0.1:42923, datanodeUuid=500d6ffd-8429-4309-bae3-992a8c7dc324, infoPort=42603, infoSecurePort=0, ipcPort=36091, storageInfo=lv=-57;cid=testClusterID;nsid=1641879641;c=1732071644397), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-20T03:00:47,814 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xfa7a41343d3a0c7c with lease ID 0xb914263cf127d1ee: Processing first storage report for DS-62d9d313-457c-499a-a1df-27558e3ed2e0 from datanode DatanodeRegistration(127.0.0.1:33951, datanodeUuid=5735bcd3-4753-44a1-aeed-94e7ab4d5722, infoPort=43923, infoSecurePort=0, ipcPort=44853, storageInfo=lv=-57;cid=testClusterID;nsid=1641879641;c=1732071644397) 2024-11-20T03:00:47,814 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xfa7a41343d3a0c7c with lease ID 0xb914263cf127d1ee: from storage DS-62d9d313-457c-499a-a1df-27558e3ed2e0 node DatanodeRegistration(127.0.0.1:33951, datanodeUuid=5735bcd3-4753-44a1-aeed-94e7ab4d5722, infoPort=43923, infoSecurePort=0, ipcPort=44853, storageInfo=lv=-57;cid=testClusterID;nsid=1641879641;c=1732071644397), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-20T03:00:47,815 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x218f085f86a807e7 with lease ID 0xb914263cf127d1ef: Processing first storage report for DS-f95d508f-7037-471b-a77d-59afbfba138c from datanode DatanodeRegistration(127.0.0.1:38873, datanodeUuid=e8cd692b-1714-45ca-b85e-b6498b320236, infoPort=38101, infoSecurePort=0, ipcPort=37647, storageInfo=lv=-57;cid=testClusterID;nsid=1641879641;c=1732071644397) 2024-11-20T03:00:47,815 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x218f085f86a807e7 with lease ID 0xb914263cf127d1ef: from storage DS-f95d508f-7037-471b-a77d-59afbfba138c node DatanodeRegistration(127.0.0.1:38873, datanodeUuid=e8cd692b-1714-45ca-b85e-b6498b320236, infoPort=38101, infoSecurePort=0, ipcPort=37647, storageInfo=lv=-57;cid=testClusterID;nsid=1641879641;c=1732071644397), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-20T03:00:47,897 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cc48e81-f8f9-9439-aadc-f08e2410e401 2024-11-20T03:00:47,967 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cc48e81-f8f9-9439-aadc-f08e2410e401/cluster_a1accffb-3df0-3830-8589-ade335756e45/zookeeper_0, clientPort=52318, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cc48e81-f8f9-9439-aadc-f08e2410e401/cluster_a1accffb-3df0-3830-8589-ade335756e45/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cc48e81-f8f9-9439-aadc-f08e2410e401/cluster_a1accffb-3df0-3830-8589-ade335756e45/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-20T03:00:47,976 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=52318 2024-11-20T03:00:47,986 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T03:00:47,989 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T03:00:48,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42923 is added to blk_1073741825_1001 (size=7) 2024-11-20T03:00:48,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33951 is added to blk_1073741825_1001 (size=7) 2024-11-20T03:00:48,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38873 is added to blk_1073741825_1001 (size=7) 2024-11-20T03:00:48,624 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766 with version=8 2024-11-20T03:00:48,624 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/hbase-staging 2024-11-20T03:00:48,714 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-20T03:00:48,947 INFO [Time-limited test {}] client.ConnectionUtils(128): master/1c82b83631a2:0 server-side Connection retries=45 2024-11-20T03:00:48,955 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T03:00:48,956 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-20T03:00:48,960 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-20T03:00:48,960 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T03:00:48,960 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-20T03:00:49,086 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-20T03:00:49,144 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-20T03:00:49,153 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-20T03:00:49,156 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-20T03:00:49,182 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 105463 (auto-detected) 2024-11-20T03:00:49,184 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-20T03:00:49,205 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39617 2024-11-20T03:00:49,226 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:39617 connecting to ZooKeeper ensemble=127.0.0.1:52318 2024-11-20T03:00:49,353 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:396170x0, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-20T03:00:49,356 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:39617-0x10157797d910000 connected 2024-11-20T03:00:49,459 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T03:00:49,462 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T03:00:49,474 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39617-0x10157797d910000, quorum=127.0.0.1:52318, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T03:00:49,477 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766, hbase.cluster.distributed=false 2024-11-20T03:00:49,498 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39617-0x10157797d910000, quorum=127.0.0.1:52318, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-20T03:00:49,502 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39617 2024-11-20T03:00:49,504 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39617 2024-11-20T03:00:49,504 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39617 2024-11-20T03:00:49,505 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39617 2024-11-20T03:00:49,506 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39617 2024-11-20T03:00:49,604 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/1c82b83631a2:0 server-side Connection retries=45 2024-11-20T03:00:49,605 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T03:00:49,605 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-20T03:00:49,606 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-20T03:00:49,606 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T03:00:49,606 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-20T03:00:49,609 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-20T03:00:49,612 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-20T03:00:49,613 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36133 2024-11-20T03:00:49,615 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:36133 connecting to ZooKeeper ensemble=127.0.0.1:52318 2024-11-20T03:00:49,616 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T03:00:49,620 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T03:00:49,656 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:361330x0, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-20T03:00:49,657 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:361330x0, quorum=127.0.0.1:52318, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T03:00:49,657 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36133-0x10157797d910001 connected 2024-11-20T03:00:49,660 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-20T03:00:49,668 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-20T03:00:49,671 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36133-0x10157797d910001, quorum=127.0.0.1:52318, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-20T03:00:49,676 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36133-0x10157797d910001, quorum=127.0.0.1:52318, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-20T03:00:49,677 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36133 2024-11-20T03:00:49,677 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36133 2024-11-20T03:00:49,678 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36133 2024-11-20T03:00:49,678 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36133 2024-11-20T03:00:49,680 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36133 2024-11-20T03:00:49,705 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/1c82b83631a2:0 server-side Connection retries=45 2024-11-20T03:00:49,705 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T03:00:49,705 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-20T03:00:49,706 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-20T03:00:49,706 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T03:00:49,706 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-20T03:00:49,706 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-20T03:00:49,707 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-20T03:00:49,708 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42975 2024-11-20T03:00:49,710 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:42975 connecting to ZooKeeper ensemble=127.0.0.1:52318 2024-11-20T03:00:49,711 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T03:00:49,716 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T03:00:49,742 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:429750x0, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-20T03:00:49,743 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:429750x0, quorum=127.0.0.1:52318, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T03:00:49,743 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:42975-0x10157797d910002 connected 2024-11-20T03:00:49,744 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-20T03:00:49,744 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-20T03:00:49,746 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42975-0x10157797d910002, quorum=127.0.0.1:52318, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-20T03:00:49,748 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42975-0x10157797d910002, quorum=127.0.0.1:52318, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-20T03:00:49,749 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42975 2024-11-20T03:00:49,749 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42975 2024-11-20T03:00:49,749 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42975 2024-11-20T03:00:49,749 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42975 2024-11-20T03:00:49,750 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42975 2024-11-20T03:00:49,766 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/1c82b83631a2:0 server-side Connection retries=45 2024-11-20T03:00:49,766 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T03:00:49,766 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-20T03:00:49,766 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-20T03:00:49,767 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T03:00:49,767 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-20T03:00:49,767 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-20T03:00:49,767 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-20T03:00:49,768 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43333 2024-11-20T03:00:49,770 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:43333 connecting to ZooKeeper ensemble=127.0.0.1:52318 2024-11-20T03:00:49,771 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T03:00:49,773 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T03:00:49,792 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:433330x0, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-20T03:00:49,793 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:433330x0, quorum=127.0.0.1:52318, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T03:00:49,793 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:43333-0x10157797d910003 connected 2024-11-20T03:00:49,794 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-20T03:00:49,798 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-20T03:00:49,799 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43333-0x10157797d910003, quorum=127.0.0.1:52318, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-20T03:00:49,801 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43333-0x10157797d910003, quorum=127.0.0.1:52318, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-20T03:00:49,802 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43333 2024-11-20T03:00:49,804 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43333 2024-11-20T03:00:49,804 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43333 2024-11-20T03:00:49,805 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43333 2024-11-20T03:00:49,805 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43333 2024-11-20T03:00:49,825 DEBUG [M:0;1c82b83631a2:39617 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;1c82b83631a2:39617 2024-11-20T03:00:49,826 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/1c82b83631a2,39617,1732071648799 2024-11-20T03:00:49,842 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43333-0x10157797d910003, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T03:00:49,842 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39617-0x10157797d910000, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T03:00:49,843 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36133-0x10157797d910001, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T03:00:49,843 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42975-0x10157797d910002, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T03:00:49,845 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39617-0x10157797d910000, quorum=127.0.0.1:52318, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/1c82b83631a2,39617,1732071648799 2024-11-20T03:00:49,875 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39617-0x10157797d910000, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T03:00:49,875 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36133-0x10157797d910001, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-20T03:00:49,875 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42975-0x10157797d910002, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-20T03:00:49,875 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43333-0x10157797d910003, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-20T03:00:49,875 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42975-0x10157797d910002, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T03:00:49,876 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43333-0x10157797d910003, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T03:00:49,876 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36133-0x10157797d910001, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T03:00:49,876 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39617-0x10157797d910000, quorum=127.0.0.1:52318, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-20T03:00:49,878 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/1c82b83631a2,39617,1732071648799 from backup master directory 2024-11-20T03:00:49,889 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42975-0x10157797d910002, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T03:00:49,889 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43333-0x10157797d910003, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T03:00:49,889 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39617-0x10157797d910000, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/1c82b83631a2,39617,1732071648799 2024-11-20T03:00:49,889 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39617-0x10157797d910000, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T03:00:49,890 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36133-0x10157797d910001, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T03:00:49,891 WARN [master/1c82b83631a2:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-20T03:00:49,892 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=1c82b83631a2,39617,1732071648799 2024-11-20T03:00:49,894 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-20T03:00:49,895 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-20T03:00:49,953 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/hbase.id] with ID: b9bce01c-e97b-4fb6-8c08-db38182824e8 2024-11-20T03:00:49,953 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/.tmp/hbase.id 2024-11-20T03:00:49,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38873 is added to blk_1073741826_1002 (size=42) 2024-11-20T03:00:49,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33951 is added to blk_1073741826_1002 (size=42) 2024-11-20T03:00:49,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42923 is added to blk_1073741826_1002 (size=42) 2024-11-20T03:00:49,970 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/.tmp/hbase.id]:[hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/hbase.id] 2024-11-20T03:00:50,022 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T03:00:50,027 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-20T03:00:50,048 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 19ms. 2024-11-20T03:00:50,077 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36133-0x10157797d910001, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T03:00:50,077 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39617-0x10157797d910000, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T03:00:50,077 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42975-0x10157797d910002, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T03:00:50,077 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43333-0x10157797d910003, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T03:00:50,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42923 is added to blk_1073741827_1003 (size=196) 2024-11-20T03:00:50,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38873 is added to blk_1073741827_1003 (size=196) 2024-11-20T03:00:50,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33951 is added to blk_1073741827_1003 (size=196) 2024-11-20T03:00:50,126 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T03:00:50,129 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-20T03:00:50,149 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T03:00:50,153 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T03:00:50,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42923 is added to blk_1073741828_1004 (size=1189) 2024-11-20T03:00:50,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38873 is added to blk_1073741828_1004 (size=1189) 2024-11-20T03:00:50,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33951 is added to blk_1073741828_1004 (size=1189) 2024-11-20T03:00:50,202 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/MasterData/data/master/store 2024-11-20T03:00:50,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33951 is added to blk_1073741829_1005 (size=34) 2024-11-20T03:00:50,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38873 is added to blk_1073741829_1005 (size=34) 2024-11-20T03:00:50,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42923 is added to blk_1073741829_1005 (size=34) 2024-11-20T03:00:50,228 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-20T03:00:50,231 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T03:00:50,232 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-20T03:00:50,232 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T03:00:50,232 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T03:00:50,234 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-20T03:00:50,234 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T03:00:50,234 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T03:00:50,235 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732071650232Disabling compacts and flushes for region at 1732071650232Disabling writes for close at 1732071650234 (+2 ms)Writing region close event to WAL at 1732071650234Closed at 1732071650234 2024-11-20T03:00:50,237 WARN [master/1c82b83631a2:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/MasterData/data/master/store/.initializing 2024-11-20T03:00:50,237 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/MasterData/WALs/1c82b83631a2,39617,1732071648799 2024-11-20T03:00:50,246 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T03:00:50,261 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=1c82b83631a2%2C39617%2C1732071648799, suffix=, logDir=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/MasterData/WALs/1c82b83631a2,39617,1732071648799, archiveDir=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/MasterData/oldWALs, maxLogs=10 2024-11-20T03:00:50,281 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/MasterData/WALs/1c82b83631a2,39617,1732071648799/1c82b83631a2%2C39617%2C1732071648799.1732071650265, exclude list is [], retry=0 2024-11-20T03:00:50,297 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33951,DS-eff30b0a-4ee9-45c4-bd81-37560890eea3,DISK] 2024-11-20T03:00:50,297 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38873,DS-afc4754b-ca55-4ef6-8e6d-da58b90deaa8,DISK] 2024-11-20T03:00:50,297 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42923,DS-26e5314a-294f-49ed-8916-15422bf99b4b,DISK] 2024-11-20T03:00:50,300 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-20T03:00:50,336 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/MasterData/WALs/1c82b83631a2,39617,1732071648799/1c82b83631a2%2C39617%2C1732071648799.1732071650265 2024-11-20T03:00:50,337 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38101:38101),(127.0.0.1/127.0.0.1:43923:43923),(127.0.0.1/127.0.0.1:42603:42603)] 2024-11-20T03:00:50,337 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-20T03:00:50,338 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T03:00:50,341 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T03:00:50,341 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T03:00:50,379 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T03:00:50,400 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-20T03:00:50,404 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T03:00:50,407 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T03:00:50,407 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T03:00:50,410 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-20T03:00:50,411 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T03:00:50,412 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T03:00:50,412 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T03:00:50,414 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-20T03:00:50,414 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T03:00:50,415 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T03:00:50,415 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T03:00:50,418 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-20T03:00:50,418 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T03:00:50,419 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T03:00:50,419 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T03:00:50,422 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-20T03:00:50,423 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-20T03:00:50,429 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T03:00:50,429 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T03:00:50,432 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-20T03:00:50,436 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T03:00:50,440 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T03:00:50,441 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61470885, jitterRate=-0.08401243388652802}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-20T03:00:50,450 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732071650355Initializing all the Stores at 1732071650357 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732071650358 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732071650358Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732071650359 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732071650359Cleaning up temporary data from old regions at 1732071650430 (+71 ms)Region opened successfully at 1732071650450 (+20 ms) 2024-11-20T03:00:50,451 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-20T03:00:50,486 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3304814d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=1c82b83631a2/172.17.0.2:0 2024-11-20T03:00:50,518 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-20T03:00:50,528 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-20T03:00:50,528 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-20T03:00:50,530 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T03:00:50,532 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-20T03:00:50,536 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-20T03:00:50,536 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-20T03:00:50,566 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-20T03:00:50,575 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39617-0x10157797d910000, quorum=127.0.0.1:52318, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-20T03:00:50,633 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-20T03:00:50,636 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-20T03:00:50,637 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39617-0x10157797d910000, quorum=127.0.0.1:52318, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-20T03:00:50,647 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-20T03:00:50,649 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-20T03:00:50,653 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39617-0x10157797d910000, quorum=127.0.0.1:52318, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-20T03:00:50,664 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-20T03:00:50,665 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39617-0x10157797d910000, quorum=127.0.0.1:52318, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-20T03:00:50,675 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-20T03:00:50,690 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39617-0x10157797d910000, quorum=127.0.0.1:52318, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-20T03:00:50,700 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-20T03:00:50,714 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39617-0x10157797d910000, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-20T03:00:50,714 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36133-0x10157797d910001, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-20T03:00:50,714 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42975-0x10157797d910002, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-20T03:00:50,714 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43333-0x10157797d910003, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-20T03:00:50,714 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39617-0x10157797d910000, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T03:00:50,714 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42975-0x10157797d910002, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T03:00:50,714 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43333-0x10157797d910003, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T03:00:50,715 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36133-0x10157797d910001, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T03:00:50,719 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=1c82b83631a2,39617,1732071648799, sessionid=0x10157797d910000, setting cluster-up flag (Was=false) 2024-11-20T03:00:50,750 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39617-0x10157797d910000, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T03:00:50,750 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36133-0x10157797d910001, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T03:00:50,750 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43333-0x10157797d910003, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T03:00:50,750 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42975-0x10157797d910002, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T03:00:50,775 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-20T03:00:50,777 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=1c82b83631a2,39617,1732071648799 2024-11-20T03:00:50,800 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39617-0x10157797d910000, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T03:00:50,800 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43333-0x10157797d910003, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T03:00:50,800 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42975-0x10157797d910002, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T03:00:50,800 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36133-0x10157797d910001, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T03:00:50,825 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-20T03:00:50,827 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=1c82b83631a2,39617,1732071648799 2024-11-20T03:00:50,835 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-20T03:00:50,900 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T03:00:50,909 INFO [RS:0;1c82b83631a2:36133 {}] regionserver.HRegionServer(746): ClusterId : b9bce01c-e97b-4fb6-8c08-db38182824e8 2024-11-20T03:00:50,909 INFO [RS:2;1c82b83631a2:43333 {}] regionserver.HRegionServer(746): ClusterId : b9bce01c-e97b-4fb6-8c08-db38182824e8 2024-11-20T03:00:50,909 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-20T03:00:50,909 INFO [RS:1;1c82b83631a2:42975 {}] regionserver.HRegionServer(746): ClusterId : b9bce01c-e97b-4fb6-8c08-db38182824e8 2024-11-20T03:00:50,911 DEBUG [RS:0;1c82b83631a2:36133 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-20T03:00:50,911 DEBUG [RS:2;1c82b83631a2:43333 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-20T03:00:50,911 DEBUG [RS:1;1c82b83631a2:42975 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-20T03:00:50,916 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-20T03:00:50,921 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 1c82b83631a2,39617,1732071648799 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-20T03:00:50,933 DEBUG [RS:1;1c82b83631a2:42975 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-20T03:00:50,933 DEBUG [RS:2;1c82b83631a2:43333 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-20T03:00:50,933 DEBUG [RS:0;1c82b83631a2:36133 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-20T03:00:50,933 DEBUG [RS:2;1c82b83631a2:43333 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-20T03:00:50,933 DEBUG [RS:1;1c82b83631a2:42975 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-20T03:00:50,933 DEBUG [RS:0;1c82b83631a2:36133 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-20T03:00:50,933 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/1c82b83631a2:0, corePoolSize=5, maxPoolSize=5 2024-11-20T03:00:50,934 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/1c82b83631a2:0, corePoolSize=5, maxPoolSize=5 2024-11-20T03:00:50,934 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/1c82b83631a2:0, corePoolSize=5, maxPoolSize=5 2024-11-20T03:00:50,934 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/1c82b83631a2:0, corePoolSize=5, maxPoolSize=5 2024-11-20T03:00:50,934 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/1c82b83631a2:0, corePoolSize=10, maxPoolSize=10 2024-11-20T03:00:50,934 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/1c82b83631a2:0, corePoolSize=1, maxPoolSize=1 2024-11-20T03:00:50,934 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/1c82b83631a2:0, corePoolSize=2, maxPoolSize=2 2024-11-20T03:00:50,934 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/1c82b83631a2:0, corePoolSize=1, maxPoolSize=1 2024-11-20T03:00:50,935 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732071680935 2024-11-20T03:00:50,937 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-20T03:00:50,937 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-20T03:00:50,939 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-20T03:00:50,940 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-20T03:00:50,941 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-20T03:00:50,941 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-20T03:00:50,942 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-20T03:00:50,942 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-20T03:00:50,943 DEBUG [RS:2;1c82b83631a2:43333 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-20T03:00:50,943 DEBUG [RS:0;1c82b83631a2:36133 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-20T03:00:50,943 DEBUG [RS:1;1c82b83631a2:42975 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-20T03:00:50,944 DEBUG [RS:1;1c82b83631a2:42975 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41c70fde, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=1c82b83631a2/172.17.0.2:0 2024-11-20T03:00:50,944 DEBUG [RS:2;1c82b83631a2:43333 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2be82e87, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=1c82b83631a2/172.17.0.2:0 2024-11-20T03:00:50,944 DEBUG [RS:0;1c82b83631a2:36133 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7dfcf615, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=1c82b83631a2/172.17.0.2:0 2024-11-20T03:00:50,948 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T03:00:50,946 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-20T03:00:50,948 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-20T03:00:50,958 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-20T03:00:50,960 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-20T03:00:50,960 DEBUG [RS:1;1c82b83631a2:42975 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;1c82b83631a2:42975 2024-11-20T03:00:50,960 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-20T03:00:50,960 DEBUG [RS:0;1c82b83631a2:36133 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;1c82b83631a2:36133 2024-11-20T03:00:50,962 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-20T03:00:50,963 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-20T03:00:50,964 INFO [RS:1;1c82b83631a2:42975 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-20T03:00:50,964 INFO [RS:0;1c82b83631a2:36133 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-20T03:00:50,964 INFO [RS:1;1c82b83631a2:42975 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-20T03:00:50,964 INFO [RS:0;1c82b83631a2:36133 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-20T03:00:50,964 DEBUG [RS:1;1c82b83631a2:42975 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-20T03:00:50,964 DEBUG [RS:0;1c82b83631a2:36133 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-20T03:00:50,966 DEBUG [RS:2;1c82b83631a2:43333 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;1c82b83631a2:43333 2024-11-20T03:00:50,966 INFO [RS:2;1c82b83631a2:43333 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-20T03:00:50,966 INFO [RS:2;1c82b83631a2:43333 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-20T03:00:50,967 DEBUG [RS:2;1c82b83631a2:43333 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-20T03:00:50,967 INFO [RS:0;1c82b83631a2:36133 {}] regionserver.HRegionServer(2659): reportForDuty to master=1c82b83631a2,39617,1732071648799 with port=36133, startcode=1732071649570 2024-11-20T03:00:50,968 INFO [RS:2;1c82b83631a2:43333 {}] regionserver.HRegionServer(2659): reportForDuty to master=1c82b83631a2,39617,1732071648799 with port=43333, startcode=1732071649765 2024-11-20T03:00:50,968 INFO [RS:1;1c82b83631a2:42975 {}] regionserver.HRegionServer(2659): reportForDuty to master=1c82b83631a2,39617,1732071648799 with port=42975, startcode=1732071649704 2024-11-20T03:00:50,978 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/1c82b83631a2:0:becomeActiveMaster-HFileCleaner.large.0-1732071650964,5,FailOnTimeoutGroup] 2024-11-20T03:00:50,979 DEBUG [RS:0;1c82b83631a2:36133 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-20T03:00:50,980 DEBUG [RS:1;1c82b83631a2:42975 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-20T03:00:50,981 DEBUG [RS:2;1c82b83631a2:43333 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-20T03:00:50,983 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/1c82b83631a2:0:becomeActiveMaster-HFileCleaner.small.0-1732071650978,5,FailOnTimeoutGroup] 2024-11-20T03:00:50,983 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-20T03:00:50,983 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-20T03:00:50,984 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-20T03:00:50,985 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-20T03:00:51,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42923 is added to blk_1073741831_1007 (size=1321) 2024-11-20T03:00:51,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33951 is added to blk_1073741831_1007 (size=1321) 2024-11-20T03:00:51,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38873 is added to blk_1073741831_1007 (size=1321) 2024-11-20T03:00:51,031 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-20T03:00:51,031 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766 2024-11-20T03:00:51,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33951 is added to blk_1073741832_1008 (size=32) 2024-11-20T03:00:51,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38873 is added to blk_1073741832_1008 (size=32) 2024-11-20T03:00:51,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42923 is added to blk_1073741832_1008 (size=32) 2024-11-20T03:00:51,048 INFO [HMasterForTest-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36443, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-11-20T03:00:51,050 INFO [HMasterForTest-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38239, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-20T03:00:51,050 INFO [HMasterForTest-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39305, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-11-20T03:00:51,053 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T03:00:51,054 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39617 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 1c82b83631a2,42975,1732071649704 2024-11-20T03:00:51,056 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-20T03:00:51,056 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39617 {}] master.ServerManager(517): Registering regionserver=1c82b83631a2,42975,1732071649704 2024-11-20T03:00:51,059 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-20T03:00:51,060 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T03:00:51,061 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T03:00:51,061 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-20T03:00:51,064 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-20T03:00:51,064 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T03:00:51,065 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T03:00:51,065 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-20T03:00:51,067 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39617 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 1c82b83631a2,36133,1732071649570 2024-11-20T03:00:51,067 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39617 {}] master.ServerManager(517): Registering regionserver=1c82b83631a2,36133,1732071649570 2024-11-20T03:00:51,068 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-20T03:00:51,069 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T03:00:51,070 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T03:00:51,070 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-20T03:00:51,072 DEBUG [RS:1;1c82b83631a2:42975 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766 2024-11-20T03:00:51,072 DEBUG [RS:1;1c82b83631a2:42975 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:38107 2024-11-20T03:00:51,072 DEBUG [RS:1;1c82b83631a2:42975 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-20T03:00:51,074 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-20T03:00:51,074 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T03:00:51,074 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39617 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 1c82b83631a2,43333,1732071649765 2024-11-20T03:00:51,074 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39617 {}] master.ServerManager(517): Registering regionserver=1c82b83631a2,43333,1732071649765 2024-11-20T03:00:51,076 DEBUG [RS:0;1c82b83631a2:36133 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766 2024-11-20T03:00:51,076 DEBUG [RS:0;1c82b83631a2:36133 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:38107 2024-11-20T03:00:51,076 DEBUG [RS:0;1c82b83631a2:36133 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-20T03:00:51,076 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T03:00:51,077 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-20T03:00:51,078 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/hbase/meta/1588230740 2024-11-20T03:00:51,079 DEBUG [RS:2;1c82b83631a2:43333 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766 2024-11-20T03:00:51,079 DEBUG [RS:2;1c82b83631a2:43333 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:38107 2024-11-20T03:00:51,079 DEBUG [RS:2;1c82b83631a2:43333 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-20T03:00:51,080 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/hbase/meta/1588230740 2024-11-20T03:00:51,083 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-20T03:00:51,083 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-20T03:00:51,084 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-20T03:00:51,087 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-20T03:00:51,091 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T03:00:51,092 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62458512, jitterRate=-0.06929564476013184}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-20T03:00:51,096 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732071651053Initializing all the Stores at 1732071651056 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732071651056Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732071651056Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732071651056Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732071651056Cleaning up temporary data from old regions at 1732071651083 (+27 ms)Region opened successfully at 1732071651096 (+13 ms) 2024-11-20T03:00:51,097 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-20T03:00:51,097 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-20T03:00:51,097 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-20T03:00:51,097 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-20T03:00:51,097 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-20T03:00:51,098 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-20T03:00:51,099 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732071651097Disabling compacts and flushes for region at 1732071651097Disabling writes for close at 1732071651097Writing region close event to WAL at 1732071651098 (+1 ms)Closed at 1732071651098 2024-11-20T03:00:51,100 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39617-0x10157797d910000, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-20T03:00:51,102 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-20T03:00:51,102 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-20T03:00:51,108 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-20T03:00:51,118 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T03:00:51,122 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-20T03:00:51,126 DEBUG [RS:1;1c82b83631a2:42975 {}] zookeeper.ZKUtil(111): regionserver:42975-0x10157797d910002, quorum=127.0.0.1:52318, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/1c82b83631a2,42975,1732071649704 2024-11-20T03:00:51,126 WARN [RS:1;1c82b83631a2:42975 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-20T03:00:51,126 DEBUG [RS:0;1c82b83631a2:36133 {}] zookeeper.ZKUtil(111): regionserver:36133-0x10157797d910001, quorum=127.0.0.1:52318, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/1c82b83631a2,36133,1732071649570 2024-11-20T03:00:51,126 DEBUG [RS:2;1c82b83631a2:43333 {}] zookeeper.ZKUtil(111): regionserver:43333-0x10157797d910003, quorum=127.0.0.1:52318, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/1c82b83631a2,43333,1732071649765 2024-11-20T03:00:51,126 WARN [RS:2;1c82b83631a2:43333 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-20T03:00:51,126 WARN [RS:0;1c82b83631a2:36133 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-20T03:00:51,126 INFO [RS:1;1c82b83631a2:42975 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T03:00:51,127 INFO [RS:0;1c82b83631a2:36133 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T03:00:51,127 INFO [RS:2;1c82b83631a2:43333 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T03:00:51,127 DEBUG [RS:1;1c82b83631a2:42975 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/WALs/1c82b83631a2,42975,1732071649704 2024-11-20T03:00:51,127 DEBUG [RS:2;1c82b83631a2:43333 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/WALs/1c82b83631a2,43333,1732071649765 2024-11-20T03:00:51,127 DEBUG [RS:0;1c82b83631a2:36133 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/WALs/1c82b83631a2,36133,1732071649570 2024-11-20T03:00:51,128 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [1c82b83631a2,36133,1732071649570] 2024-11-20T03:00:51,128 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [1c82b83631a2,43333,1732071649765] 2024-11-20T03:00:51,128 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [1c82b83631a2,42975,1732071649704] 2024-11-20T03:00:51,151 INFO [RS:2;1c82b83631a2:43333 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-20T03:00:51,151 INFO [RS:0;1c82b83631a2:36133 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-20T03:00:51,151 INFO [RS:1;1c82b83631a2:42975 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-20T03:00:51,170 INFO [RS:2;1c82b83631a2:43333 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-20T03:00:51,170 INFO [RS:0;1c82b83631a2:36133 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-20T03:00:51,171 INFO [RS:1;1c82b83631a2:42975 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-20T03:00:51,177 INFO [RS:1;1c82b83631a2:42975 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-20T03:00:51,177 INFO [RS:0;1c82b83631a2:36133 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-20T03:00:51,177 INFO [RS:0;1c82b83631a2:36133 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T03:00:51,177 INFO [RS:1;1c82b83631a2:42975 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T03:00:51,177 INFO [RS:2;1c82b83631a2:43333 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-20T03:00:51,178 INFO [RS:2;1c82b83631a2:43333 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T03:00:51,183 INFO [RS:1;1c82b83631a2:42975 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-20T03:00:51,183 INFO [RS:2;1c82b83631a2:43333 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-20T03:00:51,183 INFO [RS:0;1c82b83631a2:36133 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-20T03:00:51,189 INFO [RS:1;1c82b83631a2:42975 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-20T03:00:51,189 INFO [RS:2;1c82b83631a2:43333 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-20T03:00:51,189 INFO [RS:0;1c82b83631a2:36133 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-20T03:00:51,191 INFO [RS:0;1c82b83631a2:36133 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-20T03:00:51,191 INFO [RS:1;1c82b83631a2:42975 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-20T03:00:51,191 INFO [RS:2;1c82b83631a2:43333 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-20T03:00:51,191 DEBUG [RS:2;1c82b83631a2:43333 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/1c82b83631a2:0, corePoolSize=1, maxPoolSize=1 2024-11-20T03:00:51,191 DEBUG [RS:0;1c82b83631a2:36133 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/1c82b83631a2:0, corePoolSize=1, maxPoolSize=1 2024-11-20T03:00:51,191 DEBUG [RS:1;1c82b83631a2:42975 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/1c82b83631a2:0, corePoolSize=1, maxPoolSize=1 2024-11-20T03:00:51,191 DEBUG [RS:1;1c82b83631a2:42975 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/1c82b83631a2:0, corePoolSize=1, maxPoolSize=1 2024-11-20T03:00:51,191 DEBUG [RS:0;1c82b83631a2:36133 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/1c82b83631a2:0, corePoolSize=1, maxPoolSize=1 2024-11-20T03:00:51,191 DEBUG [RS:2;1c82b83631a2:43333 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/1c82b83631a2:0, corePoolSize=1, maxPoolSize=1 2024-11-20T03:00:51,191 DEBUG [RS:1;1c82b83631a2:42975 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/1c82b83631a2:0, corePoolSize=1, maxPoolSize=1 2024-11-20T03:00:51,191 DEBUG [RS:0;1c82b83631a2:36133 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/1c82b83631a2:0, corePoolSize=1, maxPoolSize=1 2024-11-20T03:00:51,191 DEBUG [RS:2;1c82b83631a2:43333 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/1c82b83631a2:0, corePoolSize=1, maxPoolSize=1 2024-11-20T03:00:51,191 DEBUG [RS:1;1c82b83631a2:42975 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/1c82b83631a2:0, corePoolSize=1, maxPoolSize=1 2024-11-20T03:00:51,191 DEBUG [RS:0;1c82b83631a2:36133 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/1c82b83631a2:0, corePoolSize=1, maxPoolSize=1 2024-11-20T03:00:51,191 DEBUG [RS:1;1c82b83631a2:42975 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/1c82b83631a2:0, corePoolSize=1, maxPoolSize=1 2024-11-20T03:00:51,191 DEBUG [RS:2;1c82b83631a2:43333 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/1c82b83631a2:0, corePoolSize=1, maxPoolSize=1 2024-11-20T03:00:51,192 DEBUG [RS:1;1c82b83631a2:42975 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/1c82b83631a2:0, corePoolSize=2, maxPoolSize=2 2024-11-20T03:00:51,192 DEBUG [RS:0;1c82b83631a2:36133 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/1c82b83631a2:0, corePoolSize=1, maxPoolSize=1 2024-11-20T03:00:51,192 DEBUG [RS:2;1c82b83631a2:43333 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/1c82b83631a2:0, corePoolSize=1, maxPoolSize=1 2024-11-20T03:00:51,192 DEBUG [RS:1;1c82b83631a2:42975 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/1c82b83631a2:0, corePoolSize=1, maxPoolSize=1 2024-11-20T03:00:51,192 DEBUG [RS:0;1c82b83631a2:36133 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/1c82b83631a2:0, corePoolSize=2, maxPoolSize=2 2024-11-20T03:00:51,192 DEBUG [RS:2;1c82b83631a2:43333 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/1c82b83631a2:0, corePoolSize=2, maxPoolSize=2 2024-11-20T03:00:51,192 DEBUG [RS:1;1c82b83631a2:42975 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/1c82b83631a2:0, corePoolSize=1, maxPoolSize=1 2024-11-20T03:00:51,192 DEBUG [RS:2;1c82b83631a2:43333 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/1c82b83631a2:0, corePoolSize=1, maxPoolSize=1 2024-11-20T03:00:51,192 DEBUG [RS:0;1c82b83631a2:36133 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/1c82b83631a2:0, corePoolSize=1, maxPoolSize=1 2024-11-20T03:00:51,192 DEBUG [RS:1;1c82b83631a2:42975 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/1c82b83631a2:0, corePoolSize=1, maxPoolSize=1 2024-11-20T03:00:51,192 DEBUG [RS:0;1c82b83631a2:36133 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/1c82b83631a2:0, corePoolSize=1, maxPoolSize=1 2024-11-20T03:00:51,192 DEBUG [RS:2;1c82b83631a2:43333 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/1c82b83631a2:0, corePoolSize=1, maxPoolSize=1 2024-11-20T03:00:51,192 DEBUG [RS:1;1c82b83631a2:42975 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/1c82b83631a2:0, corePoolSize=1, maxPoolSize=1 2024-11-20T03:00:51,192 DEBUG [RS:0;1c82b83631a2:36133 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/1c82b83631a2:0, corePoolSize=1, maxPoolSize=1 2024-11-20T03:00:51,192 DEBUG [RS:1;1c82b83631a2:42975 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/1c82b83631a2:0, corePoolSize=1, maxPoolSize=1 2024-11-20T03:00:51,192 DEBUG [RS:2;1c82b83631a2:43333 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/1c82b83631a2:0, corePoolSize=1, maxPoolSize=1 2024-11-20T03:00:51,192 DEBUG [RS:0;1c82b83631a2:36133 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/1c82b83631a2:0, corePoolSize=1, maxPoolSize=1 2024-11-20T03:00:51,192 DEBUG [RS:1;1c82b83631a2:42975 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/1c82b83631a2:0, corePoolSize=1, maxPoolSize=1 2024-11-20T03:00:51,192 DEBUG [RS:2;1c82b83631a2:43333 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/1c82b83631a2:0, corePoolSize=1, maxPoolSize=1 2024-11-20T03:00:51,192 DEBUG [RS:0;1c82b83631a2:36133 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/1c82b83631a2:0, corePoolSize=1, maxPoolSize=1 2024-11-20T03:00:51,193 DEBUG [RS:1;1c82b83631a2:42975 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/1c82b83631a2:0, corePoolSize=3, maxPoolSize=3 2024-11-20T03:00:51,193 DEBUG [RS:0;1c82b83631a2:36133 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/1c82b83631a2:0, corePoolSize=1, maxPoolSize=1 2024-11-20T03:00:51,193 DEBUG [RS:2;1c82b83631a2:43333 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/1c82b83631a2:0, corePoolSize=1, maxPoolSize=1 2024-11-20T03:00:51,193 DEBUG [RS:0;1c82b83631a2:36133 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/1c82b83631a2:0, corePoolSize=3, maxPoolSize=3 2024-11-20T03:00:51,193 DEBUG [RS:1;1c82b83631a2:42975 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/1c82b83631a2:0, corePoolSize=3, maxPoolSize=3 2024-11-20T03:00:51,193 DEBUG [RS:2;1c82b83631a2:43333 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/1c82b83631a2:0, corePoolSize=1, maxPoolSize=1 2024-11-20T03:00:51,193 DEBUG [RS:0;1c82b83631a2:36133 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/1c82b83631a2:0, corePoolSize=3, maxPoolSize=3 2024-11-20T03:00:51,193 DEBUG [RS:2;1c82b83631a2:43333 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/1c82b83631a2:0, corePoolSize=3, maxPoolSize=3 2024-11-20T03:00:51,193 DEBUG [RS:2;1c82b83631a2:43333 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/1c82b83631a2:0, corePoolSize=3, maxPoolSize=3 2024-11-20T03:00:51,194 INFO [RS:0;1c82b83631a2:36133 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T03:00:51,194 INFO [RS:2;1c82b83631a2:43333 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T03:00:51,194 INFO [RS:0;1c82b83631a2:36133 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T03:00:51,194 INFO [RS:1;1c82b83631a2:42975 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T03:00:51,194 INFO [RS:2;1c82b83631a2:43333 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T03:00:51,195 INFO [RS:0;1c82b83631a2:36133 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T03:00:51,195 INFO [RS:1;1c82b83631a2:42975 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T03:00:51,195 INFO [RS:2;1c82b83631a2:43333 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T03:00:51,195 INFO [RS:0;1c82b83631a2:36133 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-20T03:00:51,195 INFO [RS:1;1c82b83631a2:42975 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T03:00:51,195 INFO [RS:2;1c82b83631a2:43333 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-20T03:00:51,195 INFO [RS:0;1c82b83631a2:36133 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-20T03:00:51,195 INFO [RS:1;1c82b83631a2:42975 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-20T03:00:51,195 INFO [RS:2;1c82b83631a2:43333 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-20T03:00:51,195 INFO [RS:0;1c82b83631a2:36133 {}] hbase.ChoreService(168): Chore ScheduledChore name=1c82b83631a2,36133,1732071649570-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-20T03:00:51,195 INFO [RS:2;1c82b83631a2:43333 {}] hbase.ChoreService(168): Chore ScheduledChore name=1c82b83631a2,43333,1732071649765-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-20T03:00:51,195 INFO [RS:1;1c82b83631a2:42975 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-20T03:00:51,195 INFO [RS:1;1c82b83631a2:42975 {}] hbase.ChoreService(168): Chore ScheduledChore name=1c82b83631a2,42975,1732071649704-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-20T03:00:51,221 INFO [RS:2;1c82b83631a2:43333 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-20T03:00:51,221 INFO [RS:0;1c82b83631a2:36133 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-20T03:00:51,223 INFO [RS:2;1c82b83631a2:43333 {}] hbase.ChoreService(168): Chore ScheduledChore name=1c82b83631a2,43333,1732071649765-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T03:00:51,223 INFO [RS:0;1c82b83631a2:36133 {}] hbase.ChoreService(168): Chore ScheduledChore name=1c82b83631a2,36133,1732071649570-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T03:00:51,223 INFO [RS:2;1c82b83631a2:43333 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T03:00:51,223 INFO [RS:0;1c82b83631a2:36133 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T03:00:51,223 INFO [RS:2;1c82b83631a2:43333 {}] regionserver.Replication(171): 1c82b83631a2,43333,1732071649765 started 2024-11-20T03:00:51,224 INFO [RS:0;1c82b83631a2:36133 {}] regionserver.Replication(171): 1c82b83631a2,36133,1732071649570 started 2024-11-20T03:00:51,224 INFO [RS:1;1c82b83631a2:42975 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-20T03:00:51,224 INFO [RS:1;1c82b83631a2:42975 {}] hbase.ChoreService(168): Chore ScheduledChore name=1c82b83631a2,42975,1732071649704-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T03:00:51,225 INFO [RS:1;1c82b83631a2:42975 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T03:00:51,225 INFO [RS:1;1c82b83631a2:42975 {}] regionserver.Replication(171): 1c82b83631a2,42975,1732071649704 started 2024-11-20T03:00:51,244 INFO [RS:1;1c82b83631a2:42975 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T03:00:51,244 INFO [RS:2;1c82b83631a2:43333 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T03:00:51,244 INFO [RS:1;1c82b83631a2:42975 {}] regionserver.HRegionServer(1482): Serving as 1c82b83631a2,42975,1732071649704, RpcServer on 1c82b83631a2/172.17.0.2:42975, sessionid=0x10157797d910002 2024-11-20T03:00:51,244 INFO [RS:2;1c82b83631a2:43333 {}] regionserver.HRegionServer(1482): Serving as 1c82b83631a2,43333,1732071649765, RpcServer on 1c82b83631a2/172.17.0.2:43333, sessionid=0x10157797d910003 2024-11-20T03:00:51,245 DEBUG [RS:1;1c82b83631a2:42975 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-20T03:00:51,245 DEBUG [RS:2;1c82b83631a2:43333 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-20T03:00:51,245 DEBUG [RS:2;1c82b83631a2:43333 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 1c82b83631a2,43333,1732071649765 2024-11-20T03:00:51,245 DEBUG [RS:1;1c82b83631a2:42975 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 1c82b83631a2,42975,1732071649704 2024-11-20T03:00:51,245 DEBUG [RS:2;1c82b83631a2:43333 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '1c82b83631a2,43333,1732071649765' 2024-11-20T03:00:51,245 DEBUG [RS:1;1c82b83631a2:42975 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '1c82b83631a2,42975,1732071649704' 2024-11-20T03:00:51,245 DEBUG [RS:2;1c82b83631a2:43333 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-20T03:00:51,245 DEBUG [RS:1;1c82b83631a2:42975 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-20T03:00:51,246 DEBUG [RS:1;1c82b83631a2:42975 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-20T03:00:51,246 DEBUG [RS:2;1c82b83631a2:43333 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-20T03:00:51,247 DEBUG [RS:1;1c82b83631a2:42975 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-20T03:00:51,247 DEBUG [RS:2;1c82b83631a2:43333 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-20T03:00:51,247 DEBUG [RS:2;1c82b83631a2:43333 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-20T03:00:51,247 DEBUG [RS:1;1c82b83631a2:42975 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-20T03:00:51,247 DEBUG [RS:2;1c82b83631a2:43333 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 1c82b83631a2,43333,1732071649765 2024-11-20T03:00:51,247 DEBUG [RS:1;1c82b83631a2:42975 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 1c82b83631a2,42975,1732071649704 2024-11-20T03:00:51,247 DEBUG [RS:2;1c82b83631a2:43333 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '1c82b83631a2,43333,1732071649765' 2024-11-20T03:00:51,247 DEBUG [RS:1;1c82b83631a2:42975 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '1c82b83631a2,42975,1732071649704' 2024-11-20T03:00:51,247 DEBUG [RS:2;1c82b83631a2:43333 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-20T03:00:51,247 DEBUG [RS:1;1c82b83631a2:42975 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-20T03:00:51,247 INFO [RS:0;1c82b83631a2:36133 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T03:00:51,247 INFO [RS:0;1c82b83631a2:36133 {}] regionserver.HRegionServer(1482): Serving as 1c82b83631a2,36133,1732071649570, RpcServer on 1c82b83631a2/172.17.0.2:36133, sessionid=0x10157797d910001 2024-11-20T03:00:51,248 DEBUG [RS:0;1c82b83631a2:36133 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-20T03:00:51,248 DEBUG [RS:0;1c82b83631a2:36133 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 1c82b83631a2,36133,1732071649570 2024-11-20T03:00:51,248 DEBUG [RS:0;1c82b83631a2:36133 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '1c82b83631a2,36133,1732071649570' 2024-11-20T03:00:51,248 DEBUG [RS:1;1c82b83631a2:42975 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-20T03:00:51,248 DEBUG [RS:0;1c82b83631a2:36133 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-20T03:00:51,248 DEBUG [RS:2;1c82b83631a2:43333 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-20T03:00:51,248 DEBUG [RS:1;1c82b83631a2:42975 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-20T03:00:51,248 INFO [RS:1;1c82b83631a2:42975 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-20T03:00:51,248 DEBUG [RS:0;1c82b83631a2:36133 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-20T03:00:51,249 INFO [RS:1;1c82b83631a2:42975 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-20T03:00:51,249 DEBUG [RS:2;1c82b83631a2:43333 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-20T03:00:51,249 INFO [RS:2;1c82b83631a2:43333 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-20T03:00:51,249 INFO [RS:2;1c82b83631a2:43333 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-20T03:00:51,249 DEBUG [RS:0;1c82b83631a2:36133 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-20T03:00:51,249 DEBUG [RS:0;1c82b83631a2:36133 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-20T03:00:51,249 DEBUG [RS:0;1c82b83631a2:36133 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 1c82b83631a2,36133,1732071649570 2024-11-20T03:00:51,249 DEBUG [RS:0;1c82b83631a2:36133 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '1c82b83631a2,36133,1732071649570' 2024-11-20T03:00:51,249 DEBUG [RS:0;1c82b83631a2:36133 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-20T03:00:51,250 DEBUG [RS:0;1c82b83631a2:36133 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-20T03:00:51,250 DEBUG [RS:0;1c82b83631a2:36133 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-20T03:00:51,250 INFO [RS:0;1c82b83631a2:36133 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-20T03:00:51,250 INFO [RS:0;1c82b83631a2:36133 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-20T03:00:51,273 WARN [1c82b83631a2:39617 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-20T03:00:51,355 INFO [RS:1;1c82b83631a2:42975 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T03:00:51,355 INFO [RS:2;1c82b83631a2:43333 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T03:00:51,355 INFO [RS:0;1c82b83631a2:36133 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T03:00:51,363 INFO [RS:1;1c82b83631a2:42975 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=1c82b83631a2%2C42975%2C1732071649704, suffix=, logDir=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/WALs/1c82b83631a2,42975,1732071649704, archiveDir=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/oldWALs, maxLogs=32 2024-11-20T03:00:51,364 INFO [RS:0;1c82b83631a2:36133 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=1c82b83631a2%2C36133%2C1732071649570, suffix=, logDir=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/WALs/1c82b83631a2,36133,1732071649570, archiveDir=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/oldWALs, maxLogs=32 2024-11-20T03:00:51,364 INFO [RS:2;1c82b83631a2:43333 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=1c82b83631a2%2C43333%2C1732071649765, suffix=, logDir=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/WALs/1c82b83631a2,43333,1732071649765, archiveDir=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/oldWALs, maxLogs=32 2024-11-20T03:00:51,386 DEBUG [RS:1;1c82b83631a2:42975 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/WALs/1c82b83631a2,42975,1732071649704/1c82b83631a2%2C42975%2C1732071649704.1732071651367, exclude list is [], retry=0 2024-11-20T03:00:51,386 DEBUG [RS:0;1c82b83631a2:36133 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/WALs/1c82b83631a2,36133,1732071649570/1c82b83631a2%2C36133%2C1732071649570.1732071651367, exclude list is [], retry=0 2024-11-20T03:00:51,386 DEBUG [RS:2;1c82b83631a2:43333 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/WALs/1c82b83631a2,43333,1732071649765/1c82b83631a2%2C43333%2C1732071649765.1732071651367, exclude list is [], retry=0 2024-11-20T03:00:51,392 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42923,DS-26e5314a-294f-49ed-8916-15422bf99b4b,DISK] 2024-11-20T03:00:51,392 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38873,DS-afc4754b-ca55-4ef6-8e6d-da58b90deaa8,DISK] 2024-11-20T03:00:51,393 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33951,DS-eff30b0a-4ee9-45c4-bd81-37560890eea3,DISK] 2024-11-20T03:00:51,393 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38873,DS-afc4754b-ca55-4ef6-8e6d-da58b90deaa8,DISK] 2024-11-20T03:00:51,394 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38873,DS-afc4754b-ca55-4ef6-8e6d-da58b90deaa8,DISK] 2024-11-20T03:00:51,394 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42923,DS-26e5314a-294f-49ed-8916-15422bf99b4b,DISK] 2024-11-20T03:00:51,394 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42923,DS-26e5314a-294f-49ed-8916-15422bf99b4b,DISK] 2024-11-20T03:00:51,395 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33951,DS-eff30b0a-4ee9-45c4-bd81-37560890eea3,DISK] 2024-11-20T03:00:51,395 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33951,DS-eff30b0a-4ee9-45c4-bd81-37560890eea3,DISK] 2024-11-20T03:00:51,402 INFO [RS:2;1c82b83631a2:43333 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/WALs/1c82b83631a2,43333,1732071649765/1c82b83631a2%2C43333%2C1732071649765.1732071651367 2024-11-20T03:00:51,402 INFO [RS:1;1c82b83631a2:42975 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/WALs/1c82b83631a2,42975,1732071649704/1c82b83631a2%2C42975%2C1732071649704.1732071651367 2024-11-20T03:00:51,404 DEBUG [RS:2;1c82b83631a2:43333 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42603:42603),(127.0.0.1/127.0.0.1:38101:38101),(127.0.0.1/127.0.0.1:43923:43923)] 2024-11-20T03:00:51,404 DEBUG [RS:1;1c82b83631a2:42975 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38101:38101),(127.0.0.1/127.0.0.1:42603:42603),(127.0.0.1/127.0.0.1:43923:43923)] 2024-11-20T03:00:51,407 INFO [RS:0;1c82b83631a2:36133 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/WALs/1c82b83631a2,36133,1732071649570/1c82b83631a2%2C36133%2C1732071649570.1732071651367 2024-11-20T03:00:51,407 DEBUG [RS:0;1c82b83631a2:36133 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42603:42603),(127.0.0.1/127.0.0.1:38101:38101),(127.0.0.1/127.0.0.1:43923:43923)] 2024-11-20T03:00:51,528 DEBUG [1c82b83631a2:39617 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-11-20T03:00:51,538 DEBUG [1c82b83631a2:39617 {}] balancer.BalancerClusterState(204): Hosts are {1c82b83631a2=0} racks are {/default-rack=0} 2024-11-20T03:00:51,544 DEBUG [1c82b83631a2:39617 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-20T03:00:51,544 DEBUG [1c82b83631a2:39617 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-20T03:00:51,544 DEBUG [1c82b83631a2:39617 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-20T03:00:51,544 DEBUG [1c82b83631a2:39617 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-20T03:00:51,544 DEBUG [1c82b83631a2:39617 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-20T03:00:51,544 DEBUG [1c82b83631a2:39617 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-20T03:00:51,544 INFO [1c82b83631a2:39617 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-20T03:00:51,544 INFO [1c82b83631a2:39617 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-20T03:00:51,544 INFO [1c82b83631a2:39617 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-20T03:00:51,544 DEBUG [1c82b83631a2:39617 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-20T03:00:51,551 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=1c82b83631a2,43333,1732071649765 2024-11-20T03:00:51,557 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1c82b83631a2,43333,1732071649765, state=OPENING 2024-11-20T03:00:51,625 DEBUG [PEWorker-1 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-20T03:00:51,633 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39617-0x10157797d910000, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T03:00:51,633 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42975-0x10157797d910002, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T03:00:51,633 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36133-0x10157797d910001, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T03:00:51,633 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43333-0x10157797d910003, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T03:00:51,635 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T03:00:51,635 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T03:00:51,635 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T03:00:51,635 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T03:00:51,638 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T03:00:51,640 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=1c82b83631a2,43333,1732071649765}] 2024-11-20T03:00:51,819 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-20T03:00:51,820 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46153, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-20T03:00:51,832 INFO [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-20T03:00:51,833 INFO [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T03:00:51,833 INFO [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-20T03:00:51,838 INFO [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=1c82b83631a2%2C43333%2C1732071649765.meta, suffix=.meta, logDir=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/WALs/1c82b83631a2,43333,1732071649765, archiveDir=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/oldWALs, maxLogs=32 2024-11-20T03:00:51,857 DEBUG [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/WALs/1c82b83631a2,43333,1732071649765/1c82b83631a2%2C43333%2C1732071649765.meta.1732071651840.meta, exclude list is [], retry=0 2024-11-20T03:00:51,862 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38873,DS-afc4754b-ca55-4ef6-8e6d-da58b90deaa8,DISK] 2024-11-20T03:00:51,862 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42923,DS-26e5314a-294f-49ed-8916-15422bf99b4b,DISK] 2024-11-20T03:00:51,862 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33951,DS-eff30b0a-4ee9-45c4-bd81-37560890eea3,DISK] 2024-11-20T03:00:51,870 INFO [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/WALs/1c82b83631a2,43333,1732071649765/1c82b83631a2%2C43333%2C1732071649765.meta.1732071651840.meta 2024-11-20T03:00:51,870 DEBUG [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38101:38101),(127.0.0.1/127.0.0.1:42603:42603),(127.0.0.1/127.0.0.1:43923:43923)] 2024-11-20T03:00:51,871 DEBUG [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-20T03:00:51,873 DEBUG [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-20T03:00:51,875 DEBUG [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-20T03:00:51,881 INFO [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-20T03:00:51,886 DEBUG [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-20T03:00:51,886 DEBUG [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T03:00:51,887 DEBUG [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-20T03:00:51,887 DEBUG [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-20T03:00:51,890 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-20T03:00:51,891 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-20T03:00:51,891 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T03:00:51,892 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T03:00:51,892 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-20T03:00:51,893 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-20T03:00:51,893 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T03:00:51,894 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T03:00:51,894 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-20T03:00:51,896 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-20T03:00:51,896 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T03:00:51,897 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T03:00:51,897 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-20T03:00:51,899 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-20T03:00:51,899 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T03:00:51,900 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T03:00:51,900 DEBUG [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-20T03:00:51,902 DEBUG [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/hbase/meta/1588230740 2024-11-20T03:00:51,905 DEBUG [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/hbase/meta/1588230740 2024-11-20T03:00:51,908 DEBUG [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-20T03:00:51,908 DEBUG [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-20T03:00:51,910 DEBUG [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-20T03:00:51,914 DEBUG [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-20T03:00:51,916 INFO [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70654052, jitterRate=0.052827417850494385}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-20T03:00:51,916 DEBUG [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-20T03:00:51,917 DEBUG [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732071651887Writing region info on filesystem at 1732071651887Initializing all the Stores at 1732071651889 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732071651889Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732071651889Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732071651889Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732071651889Cleaning up temporary data from old regions at 1732071651908 (+19 ms)Running coprocessor post-open hooks at 1732071651916 (+8 ms)Region opened successfully at 1732071651917 (+1 ms) 2024-11-20T03:00:51,925 INFO [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732071651810 2024-11-20T03:00:51,939 DEBUG [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-20T03:00:51,939 INFO [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-20T03:00:51,942 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=1c82b83631a2,43333,1732071649765 2024-11-20T03:00:51,945 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1c82b83631a2,43333,1732071649765, state=OPEN 2024-11-20T03:00:51,975 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39617-0x10157797d910000, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T03:00:51,975 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43333-0x10157797d910003, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T03:00:51,975 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36133-0x10157797d910001, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T03:00:51,975 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T03:00:51,975 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T03:00:51,975 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T03:00:51,975 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42975-0x10157797d910002, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T03:00:51,975 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T03:00:51,976 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=1c82b83631a2,43333,1732071649765 2024-11-20T03:00:51,984 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-20T03:00:51,985 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1c82b83631a2,43333,1732071649765 in 337 msec 2024-11-20T03:00:51,997 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-20T03:00:51,997 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 881 msec 2024-11-20T03:00:52,000 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-20T03:00:52,000 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-20T03:00:52,026 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T03:00:52,027 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1c82b83631a2,43333,1732071649765, seqNum=-1] 2024-11-20T03:00:52,054 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T03:00:52,056 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35311, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T03:00:52,090 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.2200 sec 2024-11-20T03:00:52,091 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732071652091, completionTime=-1 2024-11-20T03:00:52,095 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-11-20T03:00:52,095 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-20T03:00:52,126 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=3 2024-11-20T03:00:52,126 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732071712126 2024-11-20T03:00:52,127 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732071772126 2024-11-20T03:00:52,127 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 31 msec 2024-11-20T03:00:52,128 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-20T03:00:52,135 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1c82b83631a2,39617,1732071648799-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T03:00:52,136 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1c82b83631a2,39617,1732071648799-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T03:00:52,136 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1c82b83631a2,39617,1732071648799-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T03:00:52,137 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-1c82b83631a2:39617, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T03:00:52,138 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-20T03:00:52,139 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-20T03:00:52,144 DEBUG [master/1c82b83631a2:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-20T03:00:52,170 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.277sec 2024-11-20T03:00:52,171 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-20T03:00:52,172 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-20T03:00:52,173 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-20T03:00:52,174 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-20T03:00:52,174 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-20T03:00:52,175 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1c82b83631a2,39617,1732071648799-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-20T03:00:52,175 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1c82b83631a2,39617,1732071648799-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-20T03:00:52,180 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-20T03:00:52,182 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-20T03:00:52,182 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1c82b83631a2,39617,1732071648799-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T03:00:52,247 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ff0a408, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T03:00:52,251 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-20T03:00:52,251 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-20T03:00:52,256 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1c82b83631a2,39617,-1 for getting cluster id 2024-11-20T03:00:52,259 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T03:00:52,274 DEBUG [HMasterForTest-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b9bce01c-e97b-4fb6-8c08-db38182824e8' 2024-11-20T03:00:52,277 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T03:00:52,277 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b9bce01c-e97b-4fb6-8c08-db38182824e8" 2024-11-20T03:00:52,278 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64712372, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T03:00:52,278 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1c82b83631a2,39617,-1] 2024-11-20T03:00:52,282 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T03:00:52,285 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T03:00:52,286 INFO [HMasterForTest-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40660, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T03:00:52,289 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6238c979, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T03:00:52,290 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T03:00:52,299 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1c82b83631a2,43333,1732071649765, seqNum=-1] 2024-11-20T03:00:52,300 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T03:00:52,304 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60808, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T03:00:52,327 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=1c82b83631a2,39617,1732071648799 2024-11-20T03:00:52,333 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-20T03:00:52,339 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 1c82b83631a2,39617,1732071648799 2024-11-20T03:00:52,342 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5c0b0600 2024-11-20T03:00:52,343 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-20T03:00:52,345 INFO [HMasterForTest-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40666, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-20T03:00:52,352 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39617 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'test', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T03:00:52,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39617 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=test 2024-11-20T03:00:52,366 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_PRE_OPERATION 2024-11-20T03:00:52,368 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39617 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "test" procId is: 4 2024-11-20T03:00:52,369 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T03:00:52,371 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-20T03:00:52,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39617 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T03:00:52,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38873 is added to blk_1073741837_1013 (size=358) 2024-11-20T03:00:52,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42923 is added to blk_1073741837_1013 (size=358) 2024-11-20T03:00:52,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33951 is added to blk_1073741837_1013 (size=358) 2024-11-20T03:00:52,400 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => f63a07d19bc4399768e352bd9fcf3523, NAME => 'test,,1732071652347.f63a07d19bc4399768e352bd9fcf3523.', STARTKEY => '', ENDKEY => 'bbb'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766 2024-11-20T03:00:52,401 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => ccd7aea6ec14848f8c6730c88ce1c780, NAME => 'test,ccc,1732071652347.ccd7aea6ec14848f8c6730c88ce1c780.', STARTKEY => 'ccc', ENDKEY => 'ddd'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766 2024-11-20T03:00:52,401 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => edd7cafc43290cb2489e8ba273907acd, NAME => 'test,bbb,1732071652347.edd7cafc43290cb2489e8ba273907acd.', STARTKEY => 'bbb', ENDKEY => 'ccc'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766 2024-11-20T03:00:52,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33951 is added to blk_1073741839_1015 (size=42) 2024-11-20T03:00:52,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42923 is added to blk_1073741839_1015 (size=42) 2024-11-20T03:00:52,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38873 is added to blk_1073741839_1015 (size=42) 2024-11-20T03:00:52,423 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,,1732071652347.f63a07d19bc4399768e352bd9fcf3523.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T03:00:52,423 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing f63a07d19bc4399768e352bd9fcf3523, disabling compactions & flushes 2024-11-20T03:00:52,423 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,,1732071652347.f63a07d19bc4399768e352bd9fcf3523. 2024-11-20T03:00:52,423 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,,1732071652347.f63a07d19bc4399768e352bd9fcf3523. 2024-11-20T03:00:52,424 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,,1732071652347.f63a07d19bc4399768e352bd9fcf3523. after waiting 0 ms 2024-11-20T03:00:52,424 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,,1732071652347.f63a07d19bc4399768e352bd9fcf3523. 2024-11-20T03:00:52,424 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,,1732071652347.f63a07d19bc4399768e352bd9fcf3523. 2024-11-20T03:00:52,424 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for f63a07d19bc4399768e352bd9fcf3523: Waiting for close lock at 1732071652423Disabling compacts and flushes for region at 1732071652423Disabling writes for close at 1732071652424 (+1 ms)Writing region close event to WAL at 1732071652424Closed at 1732071652424 2024-11-20T03:00:52,425 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 63b717291dd2cfc1f752c75b18ccc26b, NAME => 'test,ddd,1732071652347.63b717291dd2cfc1f752c75b18ccc26b.', STARTKEY => 'ddd', ENDKEY => 'eee'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766 2024-11-20T03:00:52,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42923 is added to blk_1073741840_1016 (size=45) 2024-11-20T03:00:52,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33951 is added to blk_1073741840_1016 (size=45) 2024-11-20T03:00:52,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38873 is added to blk_1073741840_1016 (size=45) 2024-11-20T03:00:52,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42923 is added to blk_1073741838_1014 (size=45) 2024-11-20T03:00:52,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33951 is added to blk_1073741838_1014 (size=45) 2024-11-20T03:00:52,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38873 is added to blk_1073741838_1014 (size=45) 2024-11-20T03:00:52,435 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,bbb,1732071652347.edd7cafc43290cb2489e8ba273907acd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T03:00:52,436 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing edd7cafc43290cb2489e8ba273907acd, disabling compactions & flushes 2024-11-20T03:00:52,436 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,bbb,1732071652347.edd7cafc43290cb2489e8ba273907acd. 2024-11-20T03:00:52,436 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,bbb,1732071652347.edd7cafc43290cb2489e8ba273907acd. 2024-11-20T03:00:52,436 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,bbb,1732071652347.edd7cafc43290cb2489e8ba273907acd. after waiting 0 ms 2024-11-20T03:00:52,436 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,bbb,1732071652347.edd7cafc43290cb2489e8ba273907acd. 2024-11-20T03:00:52,436 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,bbb,1732071652347.edd7cafc43290cb2489e8ba273907acd. 2024-11-20T03:00:52,436 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for edd7cafc43290cb2489e8ba273907acd: Waiting for close lock at 1732071652435Disabling compacts and flushes for region at 1732071652435Disabling writes for close at 1732071652436 (+1 ms)Writing region close event to WAL at 1732071652436Closed at 1732071652436 2024-11-20T03:00:52,437 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 2fe87cdaf67461ff5c508d851e418c66, NAME => 'test,eee,1732071652347.2fe87cdaf67461ff5c508d851e418c66.', STARTKEY => 'eee', ENDKEY => 'fff'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766 2024-11-20T03:00:52,439 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ccc,1732071652347.ccd7aea6ec14848f8c6730c88ce1c780.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T03:00:52,439 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing ccd7aea6ec14848f8c6730c88ce1c780, disabling compactions & flushes 2024-11-20T03:00:52,439 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ccc,1732071652347.ccd7aea6ec14848f8c6730c88ce1c780. 2024-11-20T03:00:52,440 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ccc,1732071652347.ccd7aea6ec14848f8c6730c88ce1c780. 2024-11-20T03:00:52,440 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ccc,1732071652347.ccd7aea6ec14848f8c6730c88ce1c780. after waiting 0 ms 2024-11-20T03:00:52,440 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ccc,1732071652347.ccd7aea6ec14848f8c6730c88ce1c780. 2024-11-20T03:00:52,440 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ccc,1732071652347.ccd7aea6ec14848f8c6730c88ce1c780. 2024-11-20T03:00:52,440 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for ccd7aea6ec14848f8c6730c88ce1c780: Waiting for close lock at 1732071652439Disabling compacts and flushes for region at 1732071652439Disabling writes for close at 1732071652440 (+1 ms)Writing region close event to WAL at 1732071652440Closed at 1732071652440 2024-11-20T03:00:52,441 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => fc28de84807b58cb0d6ceaa73127debb, NAME => 'test,fff,1732071652347.fc28de84807b58cb0d6ceaa73127debb.', STARTKEY => 'fff', ENDKEY => 'ggg'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766 2024-11-20T03:00:52,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42923 is added to blk_1073741841_1017 (size=45) 2024-11-20T03:00:52,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38873 is added to blk_1073741841_1017 (size=45) 2024-11-20T03:00:52,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33951 is added to blk_1073741841_1017 (size=45) 2024-11-20T03:00:52,447 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,ddd,1732071652347.63b717291dd2cfc1f752c75b18ccc26b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T03:00:52,447 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 63b717291dd2cfc1f752c75b18ccc26b, disabling compactions & flushes 2024-11-20T03:00:52,447 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,ddd,1732071652347.63b717291dd2cfc1f752c75b18ccc26b. 2024-11-20T03:00:52,447 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ddd,1732071652347.63b717291dd2cfc1f752c75b18ccc26b. 2024-11-20T03:00:52,447 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,ddd,1732071652347.63b717291dd2cfc1f752c75b18ccc26b. after waiting 0 ms 2024-11-20T03:00:52,447 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,ddd,1732071652347.63b717291dd2cfc1f752c75b18ccc26b. 2024-11-20T03:00:52,447 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,ddd,1732071652347.63b717291dd2cfc1f752c75b18ccc26b. 2024-11-20T03:00:52,448 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 63b717291dd2cfc1f752c75b18ccc26b: Waiting for close lock at 1732071652447Disabling compacts and flushes for region at 1732071652447Disabling writes for close at 1732071652447Writing region close event to WAL at 1732071652447Closed at 1732071652447 2024-11-20T03:00:52,448 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 3c5e70743a336dce38e44ee0f8059378, NAME => 'test,ggg,1732071652347.3c5e70743a336dce38e44ee0f8059378.', STARTKEY => 'ggg', ENDKEY => 'hhh'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766 2024-11-20T03:00:52,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33951 is added to blk_1073741842_1018 (size=45) 2024-11-20T03:00:52,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42923 is added to blk_1073741842_1018 (size=45) 2024-11-20T03:00:52,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38873 is added to blk_1073741842_1018 (size=45) 2024-11-20T03:00:52,464 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,eee,1732071652347.2fe87cdaf67461ff5c508d851e418c66.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T03:00:52,464 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 2fe87cdaf67461ff5c508d851e418c66, disabling compactions & flushes 2024-11-20T03:00:52,464 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,eee,1732071652347.2fe87cdaf67461ff5c508d851e418c66. 2024-11-20T03:00:52,464 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,eee,1732071652347.2fe87cdaf67461ff5c508d851e418c66. 2024-11-20T03:00:52,464 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,eee,1732071652347.2fe87cdaf67461ff5c508d851e418c66. after waiting 0 ms 2024-11-20T03:00:52,464 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,eee,1732071652347.2fe87cdaf67461ff5c508d851e418c66. 2024-11-20T03:00:52,464 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,eee,1732071652347.2fe87cdaf67461ff5c508d851e418c66. 2024-11-20T03:00:52,464 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 2fe87cdaf67461ff5c508d851e418c66: Waiting for close lock at 1732071652464Disabling compacts and flushes for region at 1732071652464Disabling writes for close at 1732071652464Writing region close event to WAL at 1732071652464Closed at 1732071652464 2024-11-20T03:00:52,465 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 2816319e15be2a1534d61304db509629, NAME => 'test,hhh,1732071652347.2816319e15be2a1534d61304db509629.', STARTKEY => 'hhh', ENDKEY => 'iii'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766 2024-11-20T03:00:52,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42923 is added to blk_1073741843_1019 (size=45) 2024-11-20T03:00:52,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33951 is added to blk_1073741843_1019 (size=45) 2024-11-20T03:00:52,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38873 is added to blk_1073741843_1019 (size=45) 2024-11-20T03:00:52,470 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,fff,1732071652347.fc28de84807b58cb0d6ceaa73127debb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T03:00:52,470 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing fc28de84807b58cb0d6ceaa73127debb, disabling compactions & flushes 2024-11-20T03:00:52,470 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,fff,1732071652347.fc28de84807b58cb0d6ceaa73127debb. 2024-11-20T03:00:52,470 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,fff,1732071652347.fc28de84807b58cb0d6ceaa73127debb. 2024-11-20T03:00:52,470 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,fff,1732071652347.fc28de84807b58cb0d6ceaa73127debb. after waiting 0 ms 2024-11-20T03:00:52,470 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,fff,1732071652347.fc28de84807b58cb0d6ceaa73127debb. 2024-11-20T03:00:52,470 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,fff,1732071652347.fc28de84807b58cb0d6ceaa73127debb. 2024-11-20T03:00:52,471 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for fc28de84807b58cb0d6ceaa73127debb: Waiting for close lock at 1732071652470Disabling compacts and flushes for region at 1732071652470Disabling writes for close at 1732071652470Writing region close event to WAL at 1732071652470Closed at 1732071652470 2024-11-20T03:00:52,471 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => f928be6f5d7a3631384ebd87c8f80996, NAME => 'test,iii,1732071652347.f928be6f5d7a3631384ebd87c8f80996.', STARTKEY => 'iii', ENDKEY => 'jjj'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766 2024-11-20T03:00:52,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42923 is added to blk_1073741844_1020 (size=45) 2024-11-20T03:00:52,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38873 is added to blk_1073741844_1020 (size=45) 2024-11-20T03:00:52,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33951 is added to blk_1073741844_1020 (size=45) 2024-11-20T03:00:52,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39617 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T03:00:52,481 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,ggg,1732071652347.3c5e70743a336dce38e44ee0f8059378.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T03:00:52,481 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 3c5e70743a336dce38e44ee0f8059378, disabling compactions & flushes 2024-11-20T03:00:52,481 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,ggg,1732071652347.3c5e70743a336dce38e44ee0f8059378. 2024-11-20T03:00:52,481 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ggg,1732071652347.3c5e70743a336dce38e44ee0f8059378. 2024-11-20T03:00:52,481 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,ggg,1732071652347.3c5e70743a336dce38e44ee0f8059378. after waiting 0 ms 2024-11-20T03:00:52,481 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,ggg,1732071652347.3c5e70743a336dce38e44ee0f8059378. 2024-11-20T03:00:52,481 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,ggg,1732071652347.3c5e70743a336dce38e44ee0f8059378. 2024-11-20T03:00:52,482 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 3c5e70743a336dce38e44ee0f8059378: Waiting for close lock at 1732071652481Disabling compacts and flushes for region at 1732071652481Disabling writes for close at 1732071652481Writing region close event to WAL at 1732071652481Closed at 1732071652481 2024-11-20T03:00:52,482 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ad7c736680f4a75df049e62d21f77c48, NAME => 'test,jjj,1732071652347.ad7c736680f4a75df049e62d21f77c48.', STARTKEY => 'jjj', ENDKEY => 'kkk'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766 2024-11-20T03:00:52,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33951 is added to blk_1073741845_1021 (size=45) 2024-11-20T03:00:52,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38873 is added to blk_1073741845_1021 (size=45) 2024-11-20T03:00:52,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42923 is added to blk_1073741845_1021 (size=45) 2024-11-20T03:00:52,495 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,hhh,1732071652347.2816319e15be2a1534d61304db509629.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T03:00:52,495 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 2816319e15be2a1534d61304db509629, disabling compactions & flushes 2024-11-20T03:00:52,495 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,hhh,1732071652347.2816319e15be2a1534d61304db509629. 2024-11-20T03:00:52,495 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,hhh,1732071652347.2816319e15be2a1534d61304db509629. 2024-11-20T03:00:52,496 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,hhh,1732071652347.2816319e15be2a1534d61304db509629. after waiting 0 ms 2024-11-20T03:00:52,496 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,hhh,1732071652347.2816319e15be2a1534d61304db509629. 2024-11-20T03:00:52,496 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,hhh,1732071652347.2816319e15be2a1534d61304db509629. 2024-11-20T03:00:52,496 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 2816319e15be2a1534d61304db509629: Waiting for close lock at 1732071652495Disabling compacts and flushes for region at 1732071652495Disabling writes for close at 1732071652496 (+1 ms)Writing region close event to WAL at 1732071652496Closed at 1732071652496 2024-11-20T03:00:52,497 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 56a64fab34378d9775e0bdf29862f7e6, NAME => 'test,kkk,1732071652347.56a64fab34378d9775e0bdf29862f7e6.', STARTKEY => 'kkk', ENDKEY => 'lll'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766 2024-11-20T03:00:52,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42923 is added to blk_1073741846_1022 (size=45) 2024-11-20T03:00:52,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33951 is added to blk_1073741846_1022 (size=45) 2024-11-20T03:00:52,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38873 is added to blk_1073741846_1022 (size=45) 2024-11-20T03:00:52,511 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,iii,1732071652347.f928be6f5d7a3631384ebd87c8f80996.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T03:00:52,511 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing f928be6f5d7a3631384ebd87c8f80996, disabling compactions & flushes 2024-11-20T03:00:52,511 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,iii,1732071652347.f928be6f5d7a3631384ebd87c8f80996. 2024-11-20T03:00:52,511 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,iii,1732071652347.f928be6f5d7a3631384ebd87c8f80996. 2024-11-20T03:00:52,511 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,iii,1732071652347.f928be6f5d7a3631384ebd87c8f80996. after waiting 0 ms 2024-11-20T03:00:52,511 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,iii,1732071652347.f928be6f5d7a3631384ebd87c8f80996. 2024-11-20T03:00:52,511 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,iii,1732071652347.f928be6f5d7a3631384ebd87c8f80996. 2024-11-20T03:00:52,512 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for f928be6f5d7a3631384ebd87c8f80996: Waiting for close lock at 1732071652511Disabling compacts and flushes for region at 1732071652511Disabling writes for close at 1732071652511Writing region close event to WAL at 1732071652511Closed at 1732071652511 2024-11-20T03:00:52,512 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 535a88ef9d35d9795c43261c431087a8, NAME => 'test,lll,1732071652347.535a88ef9d35d9795c43261c431087a8.', STARTKEY => 'lll', ENDKEY => 'mmm'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766 2024-11-20T03:00:52,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33951 is added to blk_1073741847_1023 (size=45) 2024-11-20T03:00:52,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42923 is added to blk_1073741847_1023 (size=45) 2024-11-20T03:00:52,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38873 is added to blk_1073741847_1023 (size=45) 2024-11-20T03:00:52,519 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,jjj,1732071652347.ad7c736680f4a75df049e62d21f77c48.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T03:00:52,519 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing ad7c736680f4a75df049e62d21f77c48, disabling compactions & flushes 2024-11-20T03:00:52,519 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,jjj,1732071652347.ad7c736680f4a75df049e62d21f77c48. 2024-11-20T03:00:52,520 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,jjj,1732071652347.ad7c736680f4a75df049e62d21f77c48. 2024-11-20T03:00:52,520 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,jjj,1732071652347.ad7c736680f4a75df049e62d21f77c48. after waiting 0 ms 2024-11-20T03:00:52,520 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,jjj,1732071652347.ad7c736680f4a75df049e62d21f77c48. 2024-11-20T03:00:52,520 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,jjj,1732071652347.ad7c736680f4a75df049e62d21f77c48. 2024-11-20T03:00:52,520 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for ad7c736680f4a75df049e62d21f77c48: Waiting for close lock at 1732071652519Disabling compacts and flushes for region at 1732071652519Disabling writes for close at 1732071652520 (+1 ms)Writing region close event to WAL at 1732071652520Closed at 1732071652520 2024-11-20T03:00:52,520 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 38fa6330e381ad6ba7391bf9050c289e, NAME => 'test,mmm,1732071652347.38fa6330e381ad6ba7391bf9050c289e.', STARTKEY => 'mmm', ENDKEY => 'nnn'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766 2024-11-20T03:00:52,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38873 is added to blk_1073741848_1024 (size=45) 2024-11-20T03:00:52,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33951 is added to blk_1073741848_1024 (size=45) 2024-11-20T03:00:52,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42923 is added to blk_1073741848_1024 (size=45) 2024-11-20T03:00:52,534 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,kkk,1732071652347.56a64fab34378d9775e0bdf29862f7e6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T03:00:52,534 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 56a64fab34378d9775e0bdf29862f7e6, disabling compactions & flushes 2024-11-20T03:00:52,534 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,kkk,1732071652347.56a64fab34378d9775e0bdf29862f7e6. 2024-11-20T03:00:52,534 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,kkk,1732071652347.56a64fab34378d9775e0bdf29862f7e6. 2024-11-20T03:00:52,534 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,kkk,1732071652347.56a64fab34378d9775e0bdf29862f7e6. after waiting 0 ms 2024-11-20T03:00:52,534 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,kkk,1732071652347.56a64fab34378d9775e0bdf29862f7e6. 2024-11-20T03:00:52,534 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,kkk,1732071652347.56a64fab34378d9775e0bdf29862f7e6. 2024-11-20T03:00:52,534 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 56a64fab34378d9775e0bdf29862f7e6: Waiting for close lock at 1732071652534Disabling compacts and flushes for region at 1732071652534Disabling writes for close at 1732071652534Writing region close event to WAL at 1732071652534Closed at 1732071652534 2024-11-20T03:00:52,535 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => fbf7ef74de7c98db6b37604b08e67282, NAME => 'test,nnn,1732071652347.fbf7ef74de7c98db6b37604b08e67282.', STARTKEY => 'nnn', ENDKEY => 'ooo'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766 2024-11-20T03:00:52,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33951 is added to blk_1073741849_1025 (size=45) 2024-11-20T03:00:52,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42923 is added to blk_1073741849_1025 (size=45) 2024-11-20T03:00:52,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38873 is added to blk_1073741849_1025 (size=45) 2024-11-20T03:00:52,548 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,mmm,1732071652347.38fa6330e381ad6ba7391bf9050c289e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T03:00:52,548 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 38fa6330e381ad6ba7391bf9050c289e, disabling compactions & flushes 2024-11-20T03:00:52,548 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,mmm,1732071652347.38fa6330e381ad6ba7391bf9050c289e. 2024-11-20T03:00:52,548 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,mmm,1732071652347.38fa6330e381ad6ba7391bf9050c289e. 2024-11-20T03:00:52,548 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,mmm,1732071652347.38fa6330e381ad6ba7391bf9050c289e. after waiting 0 ms 2024-11-20T03:00:52,548 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,mmm,1732071652347.38fa6330e381ad6ba7391bf9050c289e. 2024-11-20T03:00:52,548 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,mmm,1732071652347.38fa6330e381ad6ba7391bf9050c289e. 2024-11-20T03:00:52,549 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 38fa6330e381ad6ba7391bf9050c289e: Waiting for close lock at 1732071652548Disabling compacts and flushes for region at 1732071652548Disabling writes for close at 1732071652548Writing region close event to WAL at 1732071652548Closed at 1732071652548 2024-11-20T03:00:52,549 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 4327d986b4dd716142908b96606b0ad7, NAME => 'test,ooo,1732071652347.4327d986b4dd716142908b96606b0ad7.', STARTKEY => 'ooo', ENDKEY => 'ppp'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766 2024-11-20T03:00:52,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33951 is added to blk_1073741850_1026 (size=45) 2024-11-20T03:00:52,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42923 is added to blk_1073741850_1026 (size=45) 2024-11-20T03:00:52,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38873 is added to blk_1073741850_1026 (size=45) 2024-11-20T03:00:52,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42923 is added to blk_1073741851_1027 (size=45) 2024-11-20T03:00:52,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33951 is added to blk_1073741851_1027 (size=45) 2024-11-20T03:00:52,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38873 is added to blk_1073741851_1027 (size=45) 2024-11-20T03:00:52,560 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,nnn,1732071652347.fbf7ef74de7c98db6b37604b08e67282.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T03:00:52,560 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing fbf7ef74de7c98db6b37604b08e67282, disabling compactions & flushes 2024-11-20T03:00:52,560 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,nnn,1732071652347.fbf7ef74de7c98db6b37604b08e67282. 2024-11-20T03:00:52,560 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,nnn,1732071652347.fbf7ef74de7c98db6b37604b08e67282. 2024-11-20T03:00:52,560 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,nnn,1732071652347.fbf7ef74de7c98db6b37604b08e67282. after waiting 0 ms 2024-11-20T03:00:52,560 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,nnn,1732071652347.fbf7ef74de7c98db6b37604b08e67282. 2024-11-20T03:00:52,560 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,nnn,1732071652347.fbf7ef74de7c98db6b37604b08e67282. 2024-11-20T03:00:52,560 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for fbf7ef74de7c98db6b37604b08e67282: Waiting for close lock at 1732071652560Disabling compacts and flushes for region at 1732071652560Disabling writes for close at 1732071652560Writing region close event to WAL at 1732071652560Closed at 1732071652560 2024-11-20T03:00:52,561 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => ef70776250de5f4e8ad7bd24027abe63, NAME => 'test,ppp,1732071652347.ef70776250de5f4e8ad7bd24027abe63.', STARTKEY => 'ppp', ENDKEY => 'qqq'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766 2024-11-20T03:00:52,563 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,lll,1732071652347.535a88ef9d35d9795c43261c431087a8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T03:00:52,564 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 535a88ef9d35d9795c43261c431087a8, disabling compactions & flushes 2024-11-20T03:00:52,564 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,lll,1732071652347.535a88ef9d35d9795c43261c431087a8. 2024-11-20T03:00:52,564 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,lll,1732071652347.535a88ef9d35d9795c43261c431087a8. 2024-11-20T03:00:52,564 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,lll,1732071652347.535a88ef9d35d9795c43261c431087a8. after waiting 0 ms 2024-11-20T03:00:52,564 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,lll,1732071652347.535a88ef9d35d9795c43261c431087a8. 2024-11-20T03:00:52,564 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,lll,1732071652347.535a88ef9d35d9795c43261c431087a8. 2024-11-20T03:00:52,564 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 535a88ef9d35d9795c43261c431087a8: Waiting for close lock at 1732071652564Disabling compacts and flushes for region at 1732071652564Disabling writes for close at 1732071652564Writing region close event to WAL at 1732071652564Closed at 1732071652564 2024-11-20T03:00:52,565 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 30cc99ff17ee8d21b793ce798b4998ba, NAME => 'test,qqq,1732071652347.30cc99ff17ee8d21b793ce798b4998ba.', STARTKEY => 'qqq', ENDKEY => 'rrr'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766 2024-11-20T03:00:52,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42923 is added to blk_1073741852_1028 (size=45) 2024-11-20T03:00:52,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33951 is added to blk_1073741852_1028 (size=45) 2024-11-20T03:00:52,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38873 is added to blk_1073741852_1028 (size=45) 2024-11-20T03:00:52,568 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,ooo,1732071652347.4327d986b4dd716142908b96606b0ad7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T03:00:52,568 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 4327d986b4dd716142908b96606b0ad7, disabling compactions & flushes 2024-11-20T03:00:52,568 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,ooo,1732071652347.4327d986b4dd716142908b96606b0ad7. 2024-11-20T03:00:52,568 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ooo,1732071652347.4327d986b4dd716142908b96606b0ad7. 2024-11-20T03:00:52,568 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,ooo,1732071652347.4327d986b4dd716142908b96606b0ad7. after waiting 0 ms 2024-11-20T03:00:52,568 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,ooo,1732071652347.4327d986b4dd716142908b96606b0ad7. 2024-11-20T03:00:52,569 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,ooo,1732071652347.4327d986b4dd716142908b96606b0ad7. 2024-11-20T03:00:52,569 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 4327d986b4dd716142908b96606b0ad7: Waiting for close lock at 1732071652568Disabling compacts and flushes for region at 1732071652568Disabling writes for close at 1732071652568Writing region close event to WAL at 1732071652569 (+1 ms)Closed at 1732071652569 2024-11-20T03:00:52,570 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a641a5878ac82d20c96fa3dff0720791, NAME => 'test,rrr,1732071652347.a641a5878ac82d20c96fa3dff0720791.', STARTKEY => 'rrr', ENDKEY => 'sss'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766 2024-11-20T03:00:52,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38873 is added to blk_1073741853_1029 (size=45) 2024-11-20T03:00:52,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33951 is added to blk_1073741853_1029 (size=45) 2024-11-20T03:00:52,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42923 is added to blk_1073741853_1029 (size=45) 2024-11-20T03:00:52,580 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,ppp,1732071652347.ef70776250de5f4e8ad7bd24027abe63.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T03:00:52,580 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing ef70776250de5f4e8ad7bd24027abe63, disabling compactions & flushes 2024-11-20T03:00:52,580 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,ppp,1732071652347.ef70776250de5f4e8ad7bd24027abe63. 2024-11-20T03:00:52,580 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ppp,1732071652347.ef70776250de5f4e8ad7bd24027abe63. 2024-11-20T03:00:52,580 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,ppp,1732071652347.ef70776250de5f4e8ad7bd24027abe63. after waiting 0 ms 2024-11-20T03:00:52,580 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,ppp,1732071652347.ef70776250de5f4e8ad7bd24027abe63. 2024-11-20T03:00:52,580 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,ppp,1732071652347.ef70776250de5f4e8ad7bd24027abe63. 2024-11-20T03:00:52,580 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for ef70776250de5f4e8ad7bd24027abe63: Waiting for close lock at 1732071652580Disabling compacts and flushes for region at 1732071652580Disabling writes for close at 1732071652580Writing region close event to WAL at 1732071652580Closed at 1732071652580 2024-11-20T03:00:52,581 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 550bbfe722915166ce74fec36828c508, NAME => 'test,sss,1732071652347.550bbfe722915166ce74fec36828c508.', STARTKEY => 'sss', ENDKEY => 'ttt'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766 2024-11-20T03:00:52,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33951 is added to blk_1073741854_1030 (size=45) 2024-11-20T03:00:52,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42923 is added to blk_1073741854_1030 (size=45) 2024-11-20T03:00:52,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38873 is added to blk_1073741854_1030 (size=45) 2024-11-20T03:00:52,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38873 is added to blk_1073741855_1031 (size=45) 2024-11-20T03:00:52,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42923 is added to blk_1073741855_1031 (size=45) 2024-11-20T03:00:52,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33951 is added to blk_1073741855_1031 (size=45) 2024-11-20T03:00:52,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33951 is added to blk_1073741856_1032 (size=45) 2024-11-20T03:00:52,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42923 is added to blk_1073741856_1032 (size=45) 2024-11-20T03:00:52,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38873 is added to blk_1073741856_1032 (size=45) 2024-11-20T03:00:52,602 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,sss,1732071652347.550bbfe722915166ce74fec36828c508.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T03:00:52,602 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 550bbfe722915166ce74fec36828c508, disabling compactions & flushes 2024-11-20T03:00:52,602 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,sss,1732071652347.550bbfe722915166ce74fec36828c508. 2024-11-20T03:00:52,602 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,sss,1732071652347.550bbfe722915166ce74fec36828c508. 2024-11-20T03:00:52,602 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,sss,1732071652347.550bbfe722915166ce74fec36828c508. after waiting 0 ms 2024-11-20T03:00:52,602 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,sss,1732071652347.550bbfe722915166ce74fec36828c508. 2024-11-20T03:00:52,603 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,sss,1732071652347.550bbfe722915166ce74fec36828c508. 2024-11-20T03:00:52,603 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 550bbfe722915166ce74fec36828c508: Waiting for close lock at 1732071652602Disabling compacts and flushes for region at 1732071652602Disabling writes for close at 1732071652602Writing region close event to WAL at 1732071652602Closed at 1732071652602 2024-11-20T03:00:52,603 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 8a818d7f8283ee7f2b5fae844b64a8ee, NAME => 'test,ttt,1732071652347.8a818d7f8283ee7f2b5fae844b64a8ee.', STARTKEY => 'ttt', ENDKEY => 'uuu'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766 2024-11-20T03:00:52,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38873 is added to blk_1073741857_1033 (size=45) 2024-11-20T03:00:52,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42923 is added to blk_1073741857_1033 (size=45) 2024-11-20T03:00:52,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33951 is added to blk_1073741857_1033 (size=45) 2024-11-20T03:00:52,614 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,ttt,1732071652347.8a818d7f8283ee7f2b5fae844b64a8ee.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T03:00:52,614 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 8a818d7f8283ee7f2b5fae844b64a8ee, disabling compactions & flushes 2024-11-20T03:00:52,614 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,ttt,1732071652347.8a818d7f8283ee7f2b5fae844b64a8ee. 2024-11-20T03:00:52,614 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ttt,1732071652347.8a818d7f8283ee7f2b5fae844b64a8ee. 2024-11-20T03:00:52,614 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,ttt,1732071652347.8a818d7f8283ee7f2b5fae844b64a8ee. after waiting 0 ms 2024-11-20T03:00:52,614 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,ttt,1732071652347.8a818d7f8283ee7f2b5fae844b64a8ee. 2024-11-20T03:00:52,614 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,ttt,1732071652347.8a818d7f8283ee7f2b5fae844b64a8ee. 2024-11-20T03:00:52,614 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 8a818d7f8283ee7f2b5fae844b64a8ee: Waiting for close lock at 1732071652614Disabling compacts and flushes for region at 1732071652614Disabling writes for close at 1732071652614Writing region close event to WAL at 1732071652614Closed at 1732071652614 2024-11-20T03:00:52,615 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 49f9e405ee146272624d3d205a4ebf9e, NAME => 'test,uuu,1732071652347.49f9e405ee146272624d3d205a4ebf9e.', STARTKEY => 'uuu', ENDKEY => 'vvv'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766 2024-11-20T03:00:52,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38873 is added to blk_1073741858_1034 (size=45) 2024-11-20T03:00:52,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33951 is added to blk_1073741858_1034 (size=45) 2024-11-20T03:00:52,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42923 is added to blk_1073741858_1034 (size=45) 2024-11-20T03:00:52,626 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,uuu,1732071652347.49f9e405ee146272624d3d205a4ebf9e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T03:00:52,626 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 49f9e405ee146272624d3d205a4ebf9e, disabling compactions & flushes 2024-11-20T03:00:52,626 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,uuu,1732071652347.49f9e405ee146272624d3d205a4ebf9e. 2024-11-20T03:00:52,626 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,uuu,1732071652347.49f9e405ee146272624d3d205a4ebf9e. 2024-11-20T03:00:52,626 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,uuu,1732071652347.49f9e405ee146272624d3d205a4ebf9e. after waiting 0 ms 2024-11-20T03:00:52,626 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,uuu,1732071652347.49f9e405ee146272624d3d205a4ebf9e. 2024-11-20T03:00:52,626 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,uuu,1732071652347.49f9e405ee146272624d3d205a4ebf9e. 2024-11-20T03:00:52,626 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 49f9e405ee146272624d3d205a4ebf9e: Waiting for close lock at 1732071652626Disabling compacts and flushes for region at 1732071652626Disabling writes for close at 1732071652626Writing region close event to WAL at 1732071652626Closed at 1732071652626 2024-11-20T03:00:52,626 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 45e8fc1ea8df34f038aeb00318494097, NAME => 'test,vvv,1732071652347.45e8fc1ea8df34f038aeb00318494097.', STARTKEY => 'vvv', ENDKEY => 'www'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766 2024-11-20T03:00:52,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42923 is added to blk_1073741859_1035 (size=45) 2024-11-20T03:00:52,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33951 is added to blk_1073741859_1035 (size=45) 2024-11-20T03:00:52,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38873 is added to blk_1073741859_1035 (size=45) 2024-11-20T03:00:52,637 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,vvv,1732071652347.45e8fc1ea8df34f038aeb00318494097.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T03:00:52,637 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 45e8fc1ea8df34f038aeb00318494097, disabling compactions & flushes 2024-11-20T03:00:52,637 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,vvv,1732071652347.45e8fc1ea8df34f038aeb00318494097. 2024-11-20T03:00:52,637 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,vvv,1732071652347.45e8fc1ea8df34f038aeb00318494097. 2024-11-20T03:00:52,637 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,vvv,1732071652347.45e8fc1ea8df34f038aeb00318494097. after waiting 0 ms 2024-11-20T03:00:52,637 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,vvv,1732071652347.45e8fc1ea8df34f038aeb00318494097. 2024-11-20T03:00:52,637 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,vvv,1732071652347.45e8fc1ea8df34f038aeb00318494097. 2024-11-20T03:00:52,637 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 45e8fc1ea8df34f038aeb00318494097: Waiting for close lock at 1732071652637Disabling compacts and flushes for region at 1732071652637Disabling writes for close at 1732071652637Writing region close event to WAL at 1732071652637Closed at 1732071652637 2024-11-20T03:00:52,638 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 8a06021a0c54a875234d29e87253d48a, NAME => 'test,www,1732071652347.8a06021a0c54a875234d29e87253d48a.', STARTKEY => 'www', ENDKEY => 'xxx'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766 2024-11-20T03:00:52,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33951 is added to blk_1073741860_1036 (size=45) 2024-11-20T03:00:52,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38873 is added to blk_1073741860_1036 (size=45) 2024-11-20T03:00:52,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42923 is added to blk_1073741860_1036 (size=45) 2024-11-20T03:00:52,649 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,www,1732071652347.8a06021a0c54a875234d29e87253d48a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T03:00:52,649 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 8a06021a0c54a875234d29e87253d48a, disabling compactions & flushes 2024-11-20T03:00:52,649 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,www,1732071652347.8a06021a0c54a875234d29e87253d48a. 2024-11-20T03:00:52,649 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,www,1732071652347.8a06021a0c54a875234d29e87253d48a. 2024-11-20T03:00:52,649 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,www,1732071652347.8a06021a0c54a875234d29e87253d48a. after waiting 0 ms 2024-11-20T03:00:52,649 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,www,1732071652347.8a06021a0c54a875234d29e87253d48a. 2024-11-20T03:00:52,649 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,www,1732071652347.8a06021a0c54a875234d29e87253d48a. 2024-11-20T03:00:52,649 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 8a06021a0c54a875234d29e87253d48a: Waiting for close lock at 1732071652649Disabling compacts and flushes for region at 1732071652649Disabling writes for close at 1732071652649Writing region close event to WAL at 1732071652649Closed at 1732071652649 2024-11-20T03:00:52,650 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => e8e6d28d8e1806fa7d958327b87df211, NAME => 'test,xxx,1732071652347.e8e6d28d8e1806fa7d958327b87df211.', STARTKEY => 'xxx', ENDKEY => 'yyy'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766 2024-11-20T03:00:52,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33951 is added to blk_1073741861_1037 (size=45) 2024-11-20T03:00:52,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38873 is added to blk_1073741861_1037 (size=45) 2024-11-20T03:00:52,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42923 is added to blk_1073741861_1037 (size=45) 2024-11-20T03:00:52,661 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,xxx,1732071652347.e8e6d28d8e1806fa7d958327b87df211.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T03:00:52,661 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing e8e6d28d8e1806fa7d958327b87df211, disabling compactions & flushes 2024-11-20T03:00:52,661 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,xxx,1732071652347.e8e6d28d8e1806fa7d958327b87df211. 2024-11-20T03:00:52,661 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,xxx,1732071652347.e8e6d28d8e1806fa7d958327b87df211. 2024-11-20T03:00:52,661 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,xxx,1732071652347.e8e6d28d8e1806fa7d958327b87df211. after waiting 0 ms 2024-11-20T03:00:52,661 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,xxx,1732071652347.e8e6d28d8e1806fa7d958327b87df211. 2024-11-20T03:00:52,661 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,xxx,1732071652347.e8e6d28d8e1806fa7d958327b87df211. 2024-11-20T03:00:52,661 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for e8e6d28d8e1806fa7d958327b87df211: Waiting for close lock at 1732071652661Disabling compacts and flushes for region at 1732071652661Disabling writes for close at 1732071652661Writing region close event to WAL at 1732071652661Closed at 1732071652661 2024-11-20T03:00:52,662 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 295cabe34d2d3dbccccb06cf3aa5bbf6, NAME => 'test,yyy,1732071652347.295cabe34d2d3dbccccb06cf3aa5bbf6.', STARTKEY => 'yyy', ENDKEY => 'zzz'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766 2024-11-20T03:00:52,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33951 is added to blk_1073741862_1038 (size=45) 2024-11-20T03:00:52,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42923 is added to blk_1073741862_1038 (size=45) 2024-11-20T03:00:52,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38873 is added to blk_1073741862_1038 (size=45) 2024-11-20T03:00:52,672 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,yyy,1732071652347.295cabe34d2d3dbccccb06cf3aa5bbf6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T03:00:52,672 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 295cabe34d2d3dbccccb06cf3aa5bbf6, disabling compactions & flushes 2024-11-20T03:00:52,672 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,yyy,1732071652347.295cabe34d2d3dbccccb06cf3aa5bbf6. 2024-11-20T03:00:52,672 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,yyy,1732071652347.295cabe34d2d3dbccccb06cf3aa5bbf6. 2024-11-20T03:00:52,672 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,yyy,1732071652347.295cabe34d2d3dbccccb06cf3aa5bbf6. after waiting 0 ms 2024-11-20T03:00:52,672 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,yyy,1732071652347.295cabe34d2d3dbccccb06cf3aa5bbf6. 2024-11-20T03:00:52,672 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,yyy,1732071652347.295cabe34d2d3dbccccb06cf3aa5bbf6. 2024-11-20T03:00:52,673 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 295cabe34d2d3dbccccb06cf3aa5bbf6: Waiting for close lock at 1732071652672Disabling compacts and flushes for region at 1732071652672Disabling writes for close at 1732071652672Writing region close event to WAL at 1732071652672Closed at 1732071652672 2024-11-20T03:00:52,673 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => bf73c409fe50a78c430f016d1431abfb, NAME => 'test,zzz,1732071652347.bf73c409fe50a78c430f016d1431abfb.', STARTKEY => 'zzz', ENDKEY => ''}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766 2024-11-20T03:00:52,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38873 is added to blk_1073741863_1039 (size=42) 2024-11-20T03:00:52,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42923 is added to blk_1073741863_1039 (size=42) 2024-11-20T03:00:52,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33951 is added to blk_1073741863_1039 (size=42) 2024-11-20T03:00:52,684 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,zzz,1732071652347.bf73c409fe50a78c430f016d1431abfb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T03:00:52,684 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing bf73c409fe50a78c430f016d1431abfb, disabling compactions & flushes 2024-11-20T03:00:52,684 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,zzz,1732071652347.bf73c409fe50a78c430f016d1431abfb. 2024-11-20T03:00:52,684 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,zzz,1732071652347.bf73c409fe50a78c430f016d1431abfb. 2024-11-20T03:00:52,684 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,zzz,1732071652347.bf73c409fe50a78c430f016d1431abfb. after waiting 0 ms 2024-11-20T03:00:52,684 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,zzz,1732071652347.bf73c409fe50a78c430f016d1431abfb. 2024-11-20T03:00:52,684 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,zzz,1732071652347.bf73c409fe50a78c430f016d1431abfb. 2024-11-20T03:00:52,684 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for bf73c409fe50a78c430f016d1431abfb: Waiting for close lock at 1732071652684Disabling compacts and flushes for region at 1732071652684Disabling writes for close at 1732071652684Writing region close event to WAL at 1732071652684Closed at 1732071652684 2024-11-20T03:00:52,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39617 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T03:00:52,990 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,qqq,1732071652347.30cc99ff17ee8d21b793ce798b4998ba.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T03:00:52,990 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 30cc99ff17ee8d21b793ce798b4998ba, disabling compactions & flushes 2024-11-20T03:00:52,990 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,qqq,1732071652347.30cc99ff17ee8d21b793ce798b4998ba. 2024-11-20T03:00:52,990 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,qqq,1732071652347.30cc99ff17ee8d21b793ce798b4998ba. 2024-11-20T03:00:52,990 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,qqq,1732071652347.30cc99ff17ee8d21b793ce798b4998ba. after waiting 0 ms 2024-11-20T03:00:52,990 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,qqq,1732071652347.30cc99ff17ee8d21b793ce798b4998ba. 2024-11-20T03:00:52,990 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,qqq,1732071652347.30cc99ff17ee8d21b793ce798b4998ba. 2024-11-20T03:00:52,991 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 30cc99ff17ee8d21b793ce798b4998ba: Waiting for close lock at 1732071652990Disabling compacts and flushes for region at 1732071652990Disabling writes for close at 1732071652990Writing region close event to WAL at 1732071652990Closed at 1732071652990 2024-11-20T03:00:52,991 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,rrr,1732071652347.a641a5878ac82d20c96fa3dff0720791.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T03:00:52,991 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing a641a5878ac82d20c96fa3dff0720791, disabling compactions & flushes 2024-11-20T03:00:52,992 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,rrr,1732071652347.a641a5878ac82d20c96fa3dff0720791. 2024-11-20T03:00:52,992 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,rrr,1732071652347.a641a5878ac82d20c96fa3dff0720791. 2024-11-20T03:00:52,992 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,rrr,1732071652347.a641a5878ac82d20c96fa3dff0720791. after waiting 0 ms 2024-11-20T03:00:52,992 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,rrr,1732071652347.a641a5878ac82d20c96fa3dff0720791. 2024-11-20T03:00:52,992 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,rrr,1732071652347.a641a5878ac82d20c96fa3dff0720791. 2024-11-20T03:00:52,992 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for a641a5878ac82d20c96fa3dff0720791: Waiting for close lock at 1732071652991Disabling compacts and flushes for region at 1732071652991Disabling writes for close at 1732071652992 (+1 ms)Writing region close event to WAL at 1732071652992Closed at 1732071652992 2024-11-20T03:00:52,996 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_ADD_TO_META 2024-11-20T03:00:53,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39617 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T03:00:53,003 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,,1732071652347.f63a07d19bc4399768e352bd9fcf3523.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1732071652996"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732071652996"}]},"ts":"1732071652996"} 2024-11-20T03:00:53,004 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,bbb,1732071652347.edd7cafc43290cb2489e8ba273907acd.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732071652996"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732071652996"}]},"ts":"1732071652996"} 2024-11-20T03:00:53,004 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ccc,1732071652347.ccd7aea6ec14848f8c6730c88ce1c780.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732071652996"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732071652996"}]},"ts":"1732071652996"} 2024-11-20T03:00:53,004 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ddd,1732071652347.63b717291dd2cfc1f752c75b18ccc26b.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732071652996"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732071652996"}]},"ts":"1732071652996"} 2024-11-20T03:00:53,004 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,eee,1732071652347.2fe87cdaf67461ff5c508d851e418c66.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732071652996"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732071652996"}]},"ts":"1732071652996"} 2024-11-20T03:00:53,005 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,fff,1732071652347.fc28de84807b58cb0d6ceaa73127debb.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732071652996"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732071652996"}]},"ts":"1732071652996"} 2024-11-20T03:00:53,005 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ggg,1732071652347.3c5e70743a336dce38e44ee0f8059378.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732071652996"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732071652996"}]},"ts":"1732071652996"} 2024-11-20T03:00:53,005 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,hhh,1732071652347.2816319e15be2a1534d61304db509629.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732071652996"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732071652996"}]},"ts":"1732071652996"} 2024-11-20T03:00:53,005 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,iii,1732071652347.f928be6f5d7a3631384ebd87c8f80996.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732071652996"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732071652996"}]},"ts":"1732071652996"} 2024-11-20T03:00:53,006 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,jjj,1732071652347.ad7c736680f4a75df049e62d21f77c48.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732071652996"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732071652996"}]},"ts":"1732071652996"} 2024-11-20T03:00:53,006 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,kkk,1732071652347.56a64fab34378d9775e0bdf29862f7e6.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732071652996"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732071652996"}]},"ts":"1732071652996"} 2024-11-20T03:00:53,006 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,mmm,1732071652347.38fa6330e381ad6ba7391bf9050c289e.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732071652996"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732071652996"}]},"ts":"1732071652996"} 2024-11-20T03:00:53,006 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,nnn,1732071652347.fbf7ef74de7c98db6b37604b08e67282.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732071652996"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732071652996"}]},"ts":"1732071652996"} 2024-11-20T03:00:53,006 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,lll,1732071652347.535a88ef9d35d9795c43261c431087a8.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732071652996"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732071652996"}]},"ts":"1732071652996"} 2024-11-20T03:00:53,006 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ooo,1732071652347.4327d986b4dd716142908b96606b0ad7.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732071652996"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732071652996"}]},"ts":"1732071652996"} 2024-11-20T03:00:53,007 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ppp,1732071652347.ef70776250de5f4e8ad7bd24027abe63.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732071652996"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732071652996"}]},"ts":"1732071652996"} 2024-11-20T03:00:53,007 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,sss,1732071652347.550bbfe722915166ce74fec36828c508.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732071652996"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732071652996"}]},"ts":"1732071652996"} 2024-11-20T03:00:53,007 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ttt,1732071652347.8a818d7f8283ee7f2b5fae844b64a8ee.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732071652996"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732071652996"}]},"ts":"1732071652996"} 2024-11-20T03:00:53,007 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,uuu,1732071652347.49f9e405ee146272624d3d205a4ebf9e.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732071652996"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732071652996"}]},"ts":"1732071652996"} 2024-11-20T03:00:53,007 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,vvv,1732071652347.45e8fc1ea8df34f038aeb00318494097.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732071652996"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732071652996"}]},"ts":"1732071652996"} 2024-11-20T03:00:53,007 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,www,1732071652347.8a06021a0c54a875234d29e87253d48a.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732071652996"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732071652996"}]},"ts":"1732071652996"} 2024-11-20T03:00:53,007 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,xxx,1732071652347.e8e6d28d8e1806fa7d958327b87df211.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732071652996"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732071652996"}]},"ts":"1732071652996"} 2024-11-20T03:00:53,007 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,yyy,1732071652347.295cabe34d2d3dbccccb06cf3aa5bbf6.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732071652996"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732071652996"}]},"ts":"1732071652996"} 2024-11-20T03:00:53,007 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,zzz,1732071652347.bf73c409fe50a78c430f016d1431abfb.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1732071652996"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732071652996"}]},"ts":"1732071652996"} 2024-11-20T03:00:53,008 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,qqq,1732071652347.30cc99ff17ee8d21b793ce798b4998ba.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732071652996"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732071652996"}]},"ts":"1732071652996"} 2024-11-20T03:00:53,008 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,rrr,1732071652347.a641a5878ac82d20c96fa3dff0720791.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732071652996"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732071652996"}]},"ts":"1732071652996"} 2024-11-20T03:00:53,046 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 26 regions to meta. 2024-11-20T03:00:53,049 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-20T03:00:53,051 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732071653049"}]},"ts":"1732071653049"} 2024-11-20T03:00:53,056 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=test, state=ENABLING in hbase:meta 2024-11-20T03:00:53,057 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(204): Hosts are {1c82b83631a2=0} racks are {/default-rack=0} 2024-11-20T03:00:53,074 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-20T03:00:53,074 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-20T03:00:53,074 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-20T03:00:53,074 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-20T03:00:53,074 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-20T03:00:53,074 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-20T03:00:53,074 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-20T03:00:53,074 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-20T03:00:53,074 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-20T03:00:53,074 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-20T03:00:53,077 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f63a07d19bc4399768e352bd9fcf3523, ASSIGN}, {pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=edd7cafc43290cb2489e8ba273907acd, ASSIGN}, {pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ccd7aea6ec14848f8c6730c88ce1c780, ASSIGN}, {pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=63b717291dd2cfc1f752c75b18ccc26b, ASSIGN}, {pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2fe87cdaf67461ff5c508d851e418c66, ASSIGN}, {pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=fc28de84807b58cb0d6ceaa73127debb, ASSIGN}, {pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=3c5e70743a336dce38e44ee0f8059378, ASSIGN}, {pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2816319e15be2a1534d61304db509629, ASSIGN}, {pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f928be6f5d7a3631384ebd87c8f80996, ASSIGN}, {pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ad7c736680f4a75df049e62d21f77c48, ASSIGN}, {pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=56a64fab34378d9775e0bdf29862f7e6, ASSIGN}, {pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=535a88ef9d35d9795c43261c431087a8, ASSIGN}, {pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=38fa6330e381ad6ba7391bf9050c289e, ASSIGN}, {pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=fbf7ef74de7c98db6b37604b08e67282, ASSIGN}, {pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4327d986b4dd716142908b96606b0ad7, ASSIGN}, {pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ef70776250de5f4e8ad7bd24027abe63, ASSIGN}, {pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=30cc99ff17ee8d21b793ce798b4998ba, ASSIGN}, {pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a641a5878ac82d20c96fa3dff0720791, ASSIGN}, {pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=550bbfe722915166ce74fec36828c508, ASSIGN}, {pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8a818d7f8283ee7f2b5fae844b64a8ee, ASSIGN}, {pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=49f9e405ee146272624d3d205a4ebf9e, ASSIGN}, {pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=45e8fc1ea8df34f038aeb00318494097, ASSIGN}, {pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8a06021a0c54a875234d29e87253d48a, ASSIGN}, {pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=e8e6d28d8e1806fa7d958327b87df211, ASSIGN}, {pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=295cabe34d2d3dbccccb06cf3aa5bbf6, ASSIGN}, {pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=bf73c409fe50a78c430f016d1431abfb, ASSIGN}] 2024-11-20T03:00:53,086 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=bf73c409fe50a78c430f016d1431abfb, ASSIGN 2024-11-20T03:00:53,087 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=bf73c409fe50a78c430f016d1431abfb, ASSIGN; state=OFFLINE, location=1c82b83631a2,43333,1732071649765; forceNewPlan=false, retain=false 2024-11-20T03:00:53,089 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=295cabe34d2d3dbccccb06cf3aa5bbf6, ASSIGN 2024-11-20T03:00:53,090 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=295cabe34d2d3dbccccb06cf3aa5bbf6, ASSIGN; state=OFFLINE, location=1c82b83631a2,36133,1732071649570; forceNewPlan=false, retain=false 2024-11-20T03:00:53,092 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=e8e6d28d8e1806fa7d958327b87df211, ASSIGN 2024-11-20T03:00:53,093 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=e8e6d28d8e1806fa7d958327b87df211, ASSIGN; state=OFFLINE, location=1c82b83631a2,42975,1732071649704; forceNewPlan=false, retain=false 2024-11-20T03:00:53,095 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8a06021a0c54a875234d29e87253d48a, ASSIGN 2024-11-20T03:00:53,097 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=8a06021a0c54a875234d29e87253d48a, ASSIGN; state=OFFLINE, location=1c82b83631a2,43333,1732071649765; forceNewPlan=false, retain=false 2024-11-20T03:00:53,099 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=45e8fc1ea8df34f038aeb00318494097, ASSIGN 2024-11-20T03:00:53,101 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=45e8fc1ea8df34f038aeb00318494097, ASSIGN; state=OFFLINE, location=1c82b83631a2,36133,1732071649570; forceNewPlan=false, retain=false 2024-11-20T03:00:53,103 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=49f9e405ee146272624d3d205a4ebf9e, ASSIGN 2024-11-20T03:00:53,105 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=49f9e405ee146272624d3d205a4ebf9e, ASSIGN; state=OFFLINE, location=1c82b83631a2,42975,1732071649704; forceNewPlan=false, retain=false 2024-11-20T03:00:53,116 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8a818d7f8283ee7f2b5fae844b64a8ee, ASSIGN 2024-11-20T03:00:53,117 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=8a818d7f8283ee7f2b5fae844b64a8ee, ASSIGN; state=OFFLINE, location=1c82b83631a2,43333,1732071649765; forceNewPlan=false, retain=false 2024-11-20T03:00:53,119 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=550bbfe722915166ce74fec36828c508, ASSIGN 2024-11-20T03:00:53,121 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=550bbfe722915166ce74fec36828c508, ASSIGN; state=OFFLINE, location=1c82b83631a2,36133,1732071649570; forceNewPlan=false, retain=false 2024-11-20T03:00:53,123 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a641a5878ac82d20c96fa3dff0720791, ASSIGN 2024-11-20T03:00:53,125 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=a641a5878ac82d20c96fa3dff0720791, ASSIGN; state=OFFLINE, location=1c82b83631a2,36133,1732071649570; forceNewPlan=false, retain=false 2024-11-20T03:00:53,127 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=30cc99ff17ee8d21b793ce798b4998ba, ASSIGN 2024-11-20T03:00:53,129 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=30cc99ff17ee8d21b793ce798b4998ba, ASSIGN; state=OFFLINE, location=1c82b83631a2,42975,1732071649704; forceNewPlan=false, retain=false 2024-11-20T03:00:53,131 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ef70776250de5f4e8ad7bd24027abe63, ASSIGN 2024-11-20T03:00:53,133 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=ef70776250de5f4e8ad7bd24027abe63, ASSIGN; state=OFFLINE, location=1c82b83631a2,42975,1732071649704; forceNewPlan=false, retain=false 2024-11-20T03:00:53,135 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4327d986b4dd716142908b96606b0ad7, ASSIGN 2024-11-20T03:00:53,137 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=4327d986b4dd716142908b96606b0ad7, ASSIGN; state=OFFLINE, location=1c82b83631a2,43333,1732071649765; forceNewPlan=false, retain=false 2024-11-20T03:00:53,139 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=fbf7ef74de7c98db6b37604b08e67282, ASSIGN 2024-11-20T03:00:53,141 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=fbf7ef74de7c98db6b37604b08e67282, ASSIGN; state=OFFLINE, location=1c82b83631a2,42975,1732071649704; forceNewPlan=false, retain=false 2024-11-20T03:00:53,143 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=38fa6330e381ad6ba7391bf9050c289e, ASSIGN 2024-11-20T03:00:53,145 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=38fa6330e381ad6ba7391bf9050c289e, ASSIGN; state=OFFLINE, location=1c82b83631a2,43333,1732071649765; forceNewPlan=false, retain=false 2024-11-20T03:00:53,147 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=535a88ef9d35d9795c43261c431087a8, ASSIGN 2024-11-20T03:00:53,149 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=535a88ef9d35d9795c43261c431087a8, ASSIGN; state=OFFLINE, location=1c82b83631a2,36133,1732071649570; forceNewPlan=false, retain=false 2024-11-20T03:00:53,151 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=56a64fab34378d9775e0bdf29862f7e6, ASSIGN 2024-11-20T03:00:53,153 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=56a64fab34378d9775e0bdf29862f7e6, ASSIGN; state=OFFLINE, location=1c82b83631a2,36133,1732071649570; forceNewPlan=false, retain=false 2024-11-20T03:00:53,155 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ad7c736680f4a75df049e62d21f77c48, ASSIGN 2024-11-20T03:00:53,157 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=ad7c736680f4a75df049e62d21f77c48, ASSIGN; state=OFFLINE, location=1c82b83631a2,42975,1732071649704; forceNewPlan=false, retain=false 2024-11-20T03:00:53,160 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f928be6f5d7a3631384ebd87c8f80996, ASSIGN 2024-11-20T03:00:53,161 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=f928be6f5d7a3631384ebd87c8f80996, ASSIGN; state=OFFLINE, location=1c82b83631a2,43333,1732071649765; forceNewPlan=false, retain=false 2024-11-20T03:00:53,163 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2816319e15be2a1534d61304db509629, ASSIGN 2024-11-20T03:00:53,165 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=2816319e15be2a1534d61304db509629, ASSIGN; state=OFFLINE, location=1c82b83631a2,36133,1732071649570; forceNewPlan=false, retain=false 2024-11-20T03:00:53,167 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=3c5e70743a336dce38e44ee0f8059378, ASSIGN 2024-11-20T03:00:53,168 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=3c5e70743a336dce38e44ee0f8059378, ASSIGN; state=OFFLINE, location=1c82b83631a2,42975,1732071649704; forceNewPlan=false, retain=false 2024-11-20T03:00:53,170 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=fc28de84807b58cb0d6ceaa73127debb, ASSIGN 2024-11-20T03:00:53,171 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=fc28de84807b58cb0d6ceaa73127debb, ASSIGN; state=OFFLINE, location=1c82b83631a2,43333,1732071649765; forceNewPlan=false, retain=false 2024-11-20T03:00:53,172 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2fe87cdaf67461ff5c508d851e418c66, ASSIGN 2024-11-20T03:00:53,174 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=2fe87cdaf67461ff5c508d851e418c66, ASSIGN; state=OFFLINE, location=1c82b83631a2,36133,1732071649570; forceNewPlan=false, retain=false 2024-11-20T03:00:53,175 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=63b717291dd2cfc1f752c75b18ccc26b, ASSIGN 2024-11-20T03:00:53,176 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=63b717291dd2cfc1f752c75b18ccc26b, ASSIGN; state=OFFLINE, location=1c82b83631a2,42975,1732071649704; forceNewPlan=false, retain=false 2024-11-20T03:00:53,177 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ccd7aea6ec14848f8c6730c88ce1c780, ASSIGN 2024-11-20T03:00:53,178 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=ccd7aea6ec14848f8c6730c88ce1c780, ASSIGN; state=OFFLINE, location=1c82b83631a2,43333,1732071649765; forceNewPlan=false, retain=false 2024-11-20T03:00:53,180 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=edd7cafc43290cb2489e8ba273907acd, ASSIGN 2024-11-20T03:00:53,181 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=edd7cafc43290cb2489e8ba273907acd, ASSIGN; state=OFFLINE, location=1c82b83631a2,36133,1732071649570; forceNewPlan=false, retain=false 2024-11-20T03:00:53,182 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f63a07d19bc4399768e352bd9fcf3523, ASSIGN 2024-11-20T03:00:53,183 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=f63a07d19bc4399768e352bd9fcf3523, ASSIGN; state=OFFLINE, location=1c82b83631a2,42975,1732071649704; forceNewPlan=false, retain=false 2024-11-20T03:00:53,244 INFO [1c82b83631a2:39617 {}] balancer.BaseLoadBalancer(388): Reassigned 26 regions. 26 retained the pre-restart assignment. 2024-11-20T03:00:53,246 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=bf73c409fe50a78c430f016d1431abfb, regionState=OPENING, regionLocation=1c82b83631a2,43333,1732071649765 2024-11-20T03:00:53,247 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=8a06021a0c54a875234d29e87253d48a, regionState=OPENING, regionLocation=1c82b83631a2,43333,1732071649765 2024-11-20T03:00:53,247 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=8a818d7f8283ee7f2b5fae844b64a8ee, regionState=OPENING, regionLocation=1c82b83631a2,43333,1732071649765 2024-11-20T03:00:53,248 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=4327d986b4dd716142908b96606b0ad7, regionState=OPENING, regionLocation=1c82b83631a2,43333,1732071649765 2024-11-20T03:00:53,249 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=38fa6330e381ad6ba7391bf9050c289e, regionState=OPENING, regionLocation=1c82b83631a2,43333,1732071649765 2024-11-20T03:00:53,249 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=f928be6f5d7a3631384ebd87c8f80996, regionState=OPENING, regionLocation=1c82b83631a2,43333,1732071649765 2024-11-20T03:00:53,250 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=fc28de84807b58cb0d6ceaa73127debb, regionState=OPENING, regionLocation=1c82b83631a2,43333,1732071649765 2024-11-20T03:00:53,250 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=bf73c409fe50a78c430f016d1431abfb, ASSIGN because future has completed 2024-11-20T03:00:53,250 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=ccd7aea6ec14848f8c6730c88ce1c780, regionState=OPENING, regionLocation=1c82b83631a2,43333,1732071649765 2024-11-20T03:00:53,251 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=e8e6d28d8e1806fa7d958327b87df211, regionState=OPENING, regionLocation=1c82b83631a2,42975,1732071649704 2024-11-20T03:00:53,251 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure bf73c409fe50a78c430f016d1431abfb, server=1c82b83631a2,43333,1732071649765}] 2024-11-20T03:00:53,252 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=8a06021a0c54a875234d29e87253d48a, ASSIGN because future has completed 2024-11-20T03:00:53,254 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=8a818d7f8283ee7f2b5fae844b64a8ee, ASSIGN because future has completed 2024-11-20T03:00:53,256 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=4327d986b4dd716142908b96606b0ad7, ASSIGN because future has completed 2024-11-20T03:00:53,258 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=38fa6330e381ad6ba7391bf9050c289e, ASSIGN because future has completed 2024-11-20T03:00:53,261 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure 38fa6330e381ad6ba7391bf9050c289e, server=1c82b83631a2,43333,1732071649765}] 2024-11-20T03:00:53,262 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=f928be6f5d7a3631384ebd87c8f80996, ASSIGN because future has completed 2024-11-20T03:00:53,264 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=fc28de84807b58cb0d6ceaa73127debb, ASSIGN because future has completed 2024-11-20T03:00:53,266 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=ccd7aea6ec14848f8c6730c88ce1c780, ASSIGN because future has completed 2024-11-20T03:00:53,267 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=e8e6d28d8e1806fa7d958327b87df211, ASSIGN because future has completed 2024-11-20T03:00:53,268 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure e8e6d28d8e1806fa7d958327b87df211, server=1c82b83631a2,42975,1732071649704}] 2024-11-20T03:00:53,273 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure ccd7aea6ec14848f8c6730c88ce1c780, server=1c82b83631a2,43333,1732071649765}] 2024-11-20T03:00:53,277 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure fc28de84807b58cb0d6ceaa73127debb, server=1c82b83631a2,43333,1732071649765}] 2024-11-20T03:00:53,282 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure f928be6f5d7a3631384ebd87c8f80996, server=1c82b83631a2,43333,1732071649765}] 2024-11-20T03:00:53,286 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4327d986b4dd716142908b96606b0ad7, server=1c82b83631a2,43333,1732071649765}] 2024-11-20T03:00:53,291 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8a818d7f8283ee7f2b5fae844b64a8ee, server=1c82b83631a2,43333,1732071649765}] 2024-11-20T03:00:53,297 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8a06021a0c54a875234d29e87253d48a, server=1c82b83631a2,43333,1732071649765}] 2024-11-20T03:00:53,301 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=49f9e405ee146272624d3d205a4ebf9e, regionState=OPENING, regionLocation=1c82b83631a2,42975,1732071649704 2024-11-20T03:00:53,302 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=30cc99ff17ee8d21b793ce798b4998ba, regionState=OPENING, regionLocation=1c82b83631a2,42975,1732071649704 2024-11-20T03:00:53,302 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=ef70776250de5f4e8ad7bd24027abe63, regionState=OPENING, regionLocation=1c82b83631a2,42975,1732071649704 2024-11-20T03:00:53,303 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=fbf7ef74de7c98db6b37604b08e67282, regionState=OPENING, regionLocation=1c82b83631a2,42975,1732071649704 2024-11-20T03:00:53,303 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=ad7c736680f4a75df049e62d21f77c48, regionState=OPENING, regionLocation=1c82b83631a2,42975,1732071649704 2024-11-20T03:00:53,304 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=49f9e405ee146272624d3d205a4ebf9e, ASSIGN because future has completed 2024-11-20T03:00:53,304 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=3c5e70743a336dce38e44ee0f8059378, regionState=OPENING, regionLocation=1c82b83631a2,42975,1732071649704 2024-11-20T03:00:53,305 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure 49f9e405ee146272624d3d205a4ebf9e, server=1c82b83631a2,42975,1732071649704}] 2024-11-20T03:00:53,306 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=30cc99ff17ee8d21b793ce798b4998ba, ASSIGN because future has completed 2024-11-20T03:00:53,308 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=ef70776250de5f4e8ad7bd24027abe63, ASSIGN because future has completed 2024-11-20T03:00:53,310 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=fbf7ef74de7c98db6b37604b08e67282, ASSIGN because future has completed 2024-11-20T03:00:53,314 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=ad7c736680f4a75df049e62d21f77c48, ASSIGN because future has completed 2024-11-20T03:00:53,315 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure ad7c736680f4a75df049e62d21f77c48, server=1c82b83631a2,42975,1732071649704}] 2024-11-20T03:00:53,317 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=3c5e70743a336dce38e44ee0f8059378, ASSIGN because future has completed 2024-11-20T03:00:53,324 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=42, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3c5e70743a336dce38e44ee0f8059378, server=1c82b83631a2,42975,1732071649704}] 2024-11-20T03:00:53,331 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure fbf7ef74de7c98db6b37604b08e67282, server=1c82b83631a2,42975,1732071649704}] 2024-11-20T03:00:53,339 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=44, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure ef70776250de5f4e8ad7bd24027abe63, server=1c82b83631a2,42975,1732071649704}] 2024-11-20T03:00:53,346 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure 30cc99ff17ee8d21b793ce798b4998ba, server=1c82b83631a2,42975,1732071649704}] 2024-11-20T03:00:53,354 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=63b717291dd2cfc1f752c75b18ccc26b, regionState=OPENING, regionLocation=1c82b83631a2,42975,1732071649704 2024-11-20T03:00:53,355 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=f63a07d19bc4399768e352bd9fcf3523, regionState=OPENING, regionLocation=1c82b83631a2,42975,1732071649704 2024-11-20T03:00:53,355 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=295cabe34d2d3dbccccb06cf3aa5bbf6, regionState=OPENING, regionLocation=1c82b83631a2,36133,1732071649570 2024-11-20T03:00:53,357 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=45e8fc1ea8df34f038aeb00318494097, regionState=OPENING, regionLocation=1c82b83631a2,36133,1732071649570 2024-11-20T03:00:53,357 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=550bbfe722915166ce74fec36828c508, regionState=OPENING, regionLocation=1c82b83631a2,36133,1732071649570 2024-11-20T03:00:53,357 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=63b717291dd2cfc1f752c75b18ccc26b, ASSIGN because future has completed 2024-11-20T03:00:53,358 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=46, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 63b717291dd2cfc1f752c75b18ccc26b, server=1c82b83631a2,42975,1732071649704}] 2024-11-20T03:00:53,361 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=f63a07d19bc4399768e352bd9fcf3523, ASSIGN because future has completed 2024-11-20T03:00:53,364 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=295cabe34d2d3dbccccb06cf3aa5bbf6, ASSIGN because future has completed 2024-11-20T03:00:53,367 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=45e8fc1ea8df34f038aeb00318494097, ASSIGN because future has completed 2024-11-20T03:00:53,369 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=550bbfe722915166ce74fec36828c508, ASSIGN because future has completed 2024-11-20T03:00:53,370 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 550bbfe722915166ce74fec36828c508, server=1c82b83631a2,36133,1732071649570}] 2024-11-20T03:00:53,375 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=48, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 45e8fc1ea8df34f038aeb00318494097, server=1c82b83631a2,36133,1732071649570}] 2024-11-20T03:00:53,379 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 295cabe34d2d3dbccccb06cf3aa5bbf6, server=1c82b83631a2,36133,1732071649570}] 2024-11-20T03:00:53,383 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=50, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure f63a07d19bc4399768e352bd9fcf3523, server=1c82b83631a2,42975,1732071649704}] 2024-11-20T03:00:53,386 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=a641a5878ac82d20c96fa3dff0720791, regionState=OPENING, regionLocation=1c82b83631a2,36133,1732071649570 2024-11-20T03:00:53,387 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=535a88ef9d35d9795c43261c431087a8, regionState=OPENING, regionLocation=1c82b83631a2,36133,1732071649570 2024-11-20T03:00:53,388 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=56a64fab34378d9775e0bdf29862f7e6, regionState=OPENING, regionLocation=1c82b83631a2,36133,1732071649570 2024-11-20T03:00:53,388 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=2816319e15be2a1534d61304db509629, regionState=OPENING, regionLocation=1c82b83631a2,36133,1732071649570 2024-11-20T03:00:53,389 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=2fe87cdaf67461ff5c508d851e418c66, regionState=OPENING, regionLocation=1c82b83631a2,36133,1732071649570 2024-11-20T03:00:53,390 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=a641a5878ac82d20c96fa3dff0720791, ASSIGN because future has completed 2024-11-20T03:00:53,390 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure a641a5878ac82d20c96fa3dff0720791, server=1c82b83631a2,36133,1732071649570}] 2024-11-20T03:00:53,391 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=535a88ef9d35d9795c43261c431087a8, ASSIGN because future has completed 2024-11-20T03:00:53,392 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=56a64fab34378d9775e0bdf29862f7e6, ASSIGN because future has completed 2024-11-20T03:00:53,393 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=2816319e15be2a1534d61304db509629, ASSIGN because future has completed 2024-11-20T03:00:53,394 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=2fe87cdaf67461ff5c508d851e418c66, ASSIGN because future has completed 2024-11-20T03:00:53,395 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=52, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2fe87cdaf67461ff5c508d851e418c66, server=1c82b83631a2,36133,1732071649570}] 2024-11-20T03:00:53,399 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2816319e15be2a1534d61304db509629, server=1c82b83631a2,36133,1732071649570}] 2024-11-20T03:00:53,404 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=54, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure 56a64fab34378d9775e0bdf29862f7e6, server=1c82b83631a2,36133,1732071649570}] 2024-11-20T03:00:53,409 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 535a88ef9d35d9795c43261c431087a8, server=1c82b83631a2,36133,1732071649570}] 2024-11-20T03:00:53,413 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(132): Open test,zzz,1732071652347.bf73c409fe50a78c430f016d1431abfb. 2024-11-20T03:00:53,413 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7752): Opening region: {ENCODED => bf73c409fe50a78c430f016d1431abfb, NAME => 'test,zzz,1732071652347.bf73c409fe50a78c430f016d1431abfb.', STARTKEY => 'zzz', ENDKEY => ''} 2024-11-20T03:00:53,414 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test bf73c409fe50a78c430f016d1431abfb 2024-11-20T03:00:53,414 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(898): Instantiated test,zzz,1732071652347.bf73c409fe50a78c430f016d1431abfb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T03:00:53,415 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7794): checking encryption for bf73c409fe50a78c430f016d1431abfb 2024-11-20T03:00:53,415 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7797): checking classloading for bf73c409fe50a78c430f016d1431abfb 2024-11-20T03:00:53,417 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=edd7cafc43290cb2489e8ba273907acd, regionState=OPENING, regionLocation=1c82b83631a2,36133,1732071649570 2024-11-20T03:00:53,420 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=edd7cafc43290cb2489e8ba273907acd, ASSIGN because future has completed 2024-11-20T03:00:53,420 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure edd7cafc43290cb2489e8ba273907acd, server=1c82b83631a2,36133,1732071649570}] 2024-11-20T03:00:53,423 INFO [StoreOpener-bf73c409fe50a78c430f016d1431abfb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region bf73c409fe50a78c430f016d1431abfb 2024-11-20T03:00:53,423 DEBUG [RSProcedureDispatcher-pool-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-20T03:00:53,425 INFO [StoreOpener-bf73c409fe50a78c430f016d1431abfb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bf73c409fe50a78c430f016d1431abfb columnFamilyName family 2024-11-20T03:00:53,425 DEBUG [StoreOpener-bf73c409fe50a78c430f016d1431abfb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T03:00:53,426 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38665, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-20T03:00:53,426 INFO [StoreOpener-bf73c409fe50a78c430f016d1431abfb-1 {}] regionserver.HStore(327): Store=bf73c409fe50a78c430f016d1431abfb/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T03:00:53,427 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1038): replaying wal for bf73c409fe50a78c430f016d1431abfb 2024-11-20T03:00:53,428 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/default/test/bf73c409fe50a78c430f016d1431abfb 2024-11-20T03:00:53,428 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/default/test/bf73c409fe50a78c430f016d1431abfb 2024-11-20T03:00:53,429 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1048): stopping wal replay for bf73c409fe50a78c430f016d1431abfb 2024-11-20T03:00:53,429 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1060): Cleaning up temporary data for bf73c409fe50a78c430f016d1431abfb 2024-11-20T03:00:53,430 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(132): Open test,jjj,1732071652347.ad7c736680f4a75df049e62d21f77c48. 2024-11-20T03:00:53,431 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7752): Opening region: {ENCODED => ad7c736680f4a75df049e62d21f77c48, NAME => 'test,jjj,1732071652347.ad7c736680f4a75df049e62d21f77c48.', STARTKEY => 'jjj', ENDKEY => 'kkk'} 2024-11-20T03:00:53,431 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test ad7c736680f4a75df049e62d21f77c48 2024-11-20T03:00:53,431 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(898): Instantiated test,jjj,1732071652347.ad7c736680f4a75df049e62d21f77c48.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T03:00:53,431 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1093): writing seq id for bf73c409fe50a78c430f016d1431abfb 2024-11-20T03:00:53,431 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7794): checking encryption for ad7c736680f4a75df049e62d21f77c48 2024-11-20T03:00:53,431 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7797): checking classloading for ad7c736680f4a75df049e62d21f77c48 2024-11-20T03:00:53,434 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/default/test/bf73c409fe50a78c430f016d1431abfb/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T03:00:53,435 INFO [StoreOpener-ad7c736680f4a75df049e62d21f77c48-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region ad7c736680f4a75df049e62d21f77c48 2024-11-20T03:00:53,435 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1114): Opened bf73c409fe50a78c430f016d1431abfb; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74295827, jitterRate=0.10709409415721893}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T03:00:53,435 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1122): Running coprocessor post-open hooks for bf73c409fe50a78c430f016d1431abfb 2024-11-20T03:00:53,436 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1006): Region open journal for bf73c409fe50a78c430f016d1431abfb: Running coprocessor pre-open hook at 1732071653415Writing region info on filesystem at 1732071653415Initializing all the Stores at 1732071653416 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732071653416Cleaning up temporary data from old regions at 1732071653429 (+13 ms)Running coprocessor post-open hooks at 1732071653435 (+6 ms)Region opened successfully at 1732071653436 (+1 ms) 2024-11-20T03:00:53,437 INFO [StoreOpener-ad7c736680f4a75df049e62d21f77c48-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ad7c736680f4a75df049e62d21f77c48 columnFamilyName family 2024-11-20T03:00:53,437 DEBUG [StoreOpener-ad7c736680f4a75df049e62d21f77c48-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T03:00:53,438 INFO [StoreOpener-ad7c736680f4a75df049e62d21f77c48-1 {}] regionserver.HStore(327): Store=ad7c736680f4a75df049e62d21f77c48/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T03:00:53,438 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2236): Post open deploy tasks for test,zzz,1732071652347.bf73c409fe50a78c430f016d1431abfb., pid=31, masterSystemTime=1732071653406 2024-11-20T03:00:53,438 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1038): replaying wal for ad7c736680f4a75df049e62d21f77c48 2024-11-20T03:00:53,439 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/default/test/ad7c736680f4a75df049e62d21f77c48 2024-11-20T03:00:53,440 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/default/test/ad7c736680f4a75df049e62d21f77c48 2024-11-20T03:00:53,440 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1048): stopping wal replay for ad7c736680f4a75df049e62d21f77c48 2024-11-20T03:00:53,440 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1060): Cleaning up temporary data for ad7c736680f4a75df049e62d21f77c48 2024-11-20T03:00:53,441 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2266): Finished post open deploy task for test,zzz,1732071652347.bf73c409fe50a78c430f016d1431abfb. 2024-11-20T03:00:53,441 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(153): Opened test,zzz,1732071652347.bf73c409fe50a78c430f016d1431abfb. 2024-11-20T03:00:53,442 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(132): Open test,ttt,1732071652347.8a818d7f8283ee7f2b5fae844b64a8ee. 2024-11-20T03:00:53,442 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7752): Opening region: {ENCODED => 8a818d7f8283ee7f2b5fae844b64a8ee, NAME => 'test,ttt,1732071652347.8a818d7f8283ee7f2b5fae844b64a8ee.', STARTKEY => 'ttt', ENDKEY => 'uuu'} 2024-11-20T03:00:53,442 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 8a818d7f8283ee7f2b5fae844b64a8ee 2024-11-20T03:00:53,443 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(898): Instantiated test,ttt,1732071652347.8a818d7f8283ee7f2b5fae844b64a8ee.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T03:00:53,443 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7794): checking encryption for 8a818d7f8283ee7f2b5fae844b64a8ee 2024-11-20T03:00:53,443 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7797): checking classloading for 8a818d7f8283ee7f2b5fae844b64a8ee 2024-11-20T03:00:53,443 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=bf73c409fe50a78c430f016d1431abfb, regionState=OPEN, openSeqNum=2, regionLocation=1c82b83631a2,43333,1732071649765 2024-11-20T03:00:53,446 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1093): writing seq id for ad7c736680f4a75df049e62d21f77c48 2024-11-20T03:00:53,447 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure bf73c409fe50a78c430f016d1431abfb, server=1c82b83631a2,43333,1732071649765 because future has completed 2024-11-20T03:00:53,449 INFO [StoreOpener-8a818d7f8283ee7f2b5fae844b64a8ee-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 8a818d7f8283ee7f2b5fae844b64a8ee 2024-11-20T03:00:53,451 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/default/test/ad7c736680f4a75df049e62d21f77c48/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T03:00:53,452 INFO [StoreOpener-8a818d7f8283ee7f2b5fae844b64a8ee-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8a818d7f8283ee7f2b5fae844b64a8ee columnFamilyName family 2024-11-20T03:00:53,452 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1114): Opened ad7c736680f4a75df049e62d21f77c48; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67294812, jitterRate=0.002770841121673584}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T03:00:53,452 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ad7c736680f4a75df049e62d21f77c48 2024-11-20T03:00:53,452 DEBUG [StoreOpener-8a818d7f8283ee7f2b5fae844b64a8ee-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T03:00:53,452 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1006): Region open journal for ad7c736680f4a75df049e62d21f77c48: Running coprocessor pre-open hook at 1732071653432Writing region info on filesystem at 1732071653432Initializing all the Stores at 1732071653433 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732071653433Cleaning up temporary data from old regions at 1732071653440 (+7 ms)Running coprocessor post-open hooks at 1732071653452 (+12 ms)Region opened successfully at 1732071653452 2024-11-20T03:00:53,453 INFO [StoreOpener-8a818d7f8283ee7f2b5fae844b64a8ee-1 {}] regionserver.HStore(327): Store=8a818d7f8283ee7f2b5fae844b64a8ee/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T03:00:53,453 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-11-20T03:00:53,453 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; OpenRegionProcedure bf73c409fe50a78c430f016d1431abfb, server=1c82b83631a2,43333,1732071649765 in 197 msec 2024-11-20T03:00:53,453 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1038): replaying wal for 8a818d7f8283ee7f2b5fae844b64a8ee 2024-11-20T03:00:53,454 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2236): Post open deploy tasks for test,jjj,1732071652347.ad7c736680f4a75df049e62d21f77c48., pid=41, masterSystemTime=1732071653423 2024-11-20T03:00:53,455 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/default/test/8a818d7f8283ee7f2b5fae844b64a8ee 2024-11-20T03:00:53,456 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/default/test/8a818d7f8283ee7f2b5fae844b64a8ee 2024-11-20T03:00:53,456 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1048): stopping wal replay for 8a818d7f8283ee7f2b5fae844b64a8ee 2024-11-20T03:00:53,457 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1060): Cleaning up temporary data for 8a818d7f8283ee7f2b5fae844b64a8ee 2024-11-20T03:00:53,459 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1093): writing seq id for 8a818d7f8283ee7f2b5fae844b64a8ee 2024-11-20T03:00:53,460 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2266): Finished post open deploy task for test,jjj,1732071652347.ad7c736680f4a75df049e62d21f77c48. 2024-11-20T03:00:53,460 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(153): Opened test,jjj,1732071652347.ad7c736680f4a75df049e62d21f77c48. 2024-11-20T03:00:53,460 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(132): Open test,nnn,1732071652347.fbf7ef74de7c98db6b37604b08e67282. 2024-11-20T03:00:53,460 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7752): Opening region: {ENCODED => fbf7ef74de7c98db6b37604b08e67282, NAME => 'test,nnn,1732071652347.fbf7ef74de7c98db6b37604b08e67282.', STARTKEY => 'nnn', ENDKEY => 'ooo'} 2024-11-20T03:00:53,461 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test fbf7ef74de7c98db6b37604b08e67282 2024-11-20T03:00:53,461 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(898): Instantiated test,nnn,1732071652347.fbf7ef74de7c98db6b37604b08e67282.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T03:00:53,461 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7794): checking encryption for fbf7ef74de7c98db6b37604b08e67282 2024-11-20T03:00:53,461 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7797): checking classloading for fbf7ef74de7c98db6b37604b08e67282 2024-11-20T03:00:53,462 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=bf73c409fe50a78c430f016d1431abfb, ASSIGN in 377 msec 2024-11-20T03:00:53,464 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=ad7c736680f4a75df049e62d21f77c48, regionState=OPEN, openSeqNum=2, regionLocation=1c82b83631a2,42975,1732071649704 2024-11-20T03:00:53,467 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure ad7c736680f4a75df049e62d21f77c48, server=1c82b83631a2,42975,1732071649704 because future has completed 2024-11-20T03:00:53,467 INFO [StoreOpener-fbf7ef74de7c98db6b37604b08e67282-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region fbf7ef74de7c98db6b37604b08e67282 2024-11-20T03:00:53,467 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/default/test/8a818d7f8283ee7f2b5fae844b64a8ee/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T03:00:53,469 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1114): Opened 8a818d7f8283ee7f2b5fae844b64a8ee; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65786407, jitterRate=-0.01970614492893219}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T03:00:53,469 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8a818d7f8283ee7f2b5fae844b64a8ee 2024-11-20T03:00:53,469 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1006): Region open journal for 8a818d7f8283ee7f2b5fae844b64a8ee: Running coprocessor pre-open hook at 1732071653443Writing region info on filesystem at 1732071653443Initializing all the Stores at 1732071653444 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732071653444Cleaning up temporary data from old regions at 1732071653457 (+13 ms)Running coprocessor post-open hooks at 1732071653469 (+12 ms)Region opened successfully at 1732071653469 2024-11-20T03:00:53,471 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ttt,1732071652347.8a818d7f8283ee7f2b5fae844b64a8ee., pid=38, masterSystemTime=1732071653406 2024-11-20T03:00:53,472 INFO [StoreOpener-fbf7ef74de7c98db6b37604b08e67282-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fbf7ef74de7c98db6b37604b08e67282 columnFamilyName family 2024-11-20T03:00:53,473 DEBUG [StoreOpener-fbf7ef74de7c98db6b37604b08e67282-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T03:00:53,474 INFO [StoreOpener-fbf7ef74de7c98db6b37604b08e67282-1 {}] regionserver.HStore(327): Store=fbf7ef74de7c98db6b37604b08e67282/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T03:00:53,474 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=14 2024-11-20T03:00:53,474 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=14, state=SUCCESS, hasLock=false; OpenRegionProcedure ad7c736680f4a75df049e62d21f77c48, server=1c82b83631a2,42975,1732071649704 in 154 msec 2024-11-20T03:00:53,474 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1038): replaying wal for fbf7ef74de7c98db6b37604b08e67282 2024-11-20T03:00:53,474 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ttt,1732071652347.8a818d7f8283ee7f2b5fae844b64a8ee. 2024-11-20T03:00:53,474 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(153): Opened test,ttt,1732071652347.8a818d7f8283ee7f2b5fae844b64a8ee. 2024-11-20T03:00:53,475 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(132): Open test,fff,1732071652347.fc28de84807b58cb0d6ceaa73127debb. 2024-11-20T03:00:53,475 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7752): Opening region: {ENCODED => fc28de84807b58cb0d6ceaa73127debb, NAME => 'test,fff,1732071652347.fc28de84807b58cb0d6ceaa73127debb.', STARTKEY => 'fff', ENDKEY => 'ggg'} 2024-11-20T03:00:53,475 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/default/test/fbf7ef74de7c98db6b37604b08e67282 2024-11-20T03:00:53,476 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test fc28de84807b58cb0d6ceaa73127debb 2024-11-20T03:00:53,476 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(898): Instantiated test,fff,1732071652347.fc28de84807b58cb0d6ceaa73127debb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T03:00:53,476 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=8a818d7f8283ee7f2b5fae844b64a8ee, regionState=OPEN, openSeqNum=2, regionLocation=1c82b83631a2,43333,1732071649765 2024-11-20T03:00:53,476 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7794): checking encryption for fc28de84807b58cb0d6ceaa73127debb 2024-11-20T03:00:53,476 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7797): checking classloading for fc28de84807b58cb0d6ceaa73127debb 2024-11-20T03:00:53,476 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/default/test/fbf7ef74de7c98db6b37604b08e67282 2024-11-20T03:00:53,477 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1048): stopping wal replay for fbf7ef74de7c98db6b37604b08e67282 2024-11-20T03:00:53,477 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1060): Cleaning up temporary data for fbf7ef74de7c98db6b37604b08e67282 2024-11-20T03:00:53,481 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8a818d7f8283ee7f2b5fae844b64a8ee, server=1c82b83631a2,43333,1732071649765 because future has completed 2024-11-20T03:00:53,482 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1093): writing seq id for fbf7ef74de7c98db6b37604b08e67282 2024-11-20T03:00:53,483 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=ad7c736680f4a75df049e62d21f77c48, ASSIGN in 404 msec 2024-11-20T03:00:53,484 INFO [StoreOpener-fc28de84807b58cb0d6ceaa73127debb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region fc28de84807b58cb0d6ceaa73127debb 2024-11-20T03:00:53,486 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/default/test/fbf7ef74de7c98db6b37604b08e67282/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T03:00:53,487 INFO [StoreOpener-fc28de84807b58cb0d6ceaa73127debb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fc28de84807b58cb0d6ceaa73127debb columnFamilyName family 2024-11-20T03:00:53,487 DEBUG [StoreOpener-fc28de84807b58cb0d6ceaa73127debb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T03:00:53,487 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1114): Opened fbf7ef74de7c98db6b37604b08e67282; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63305891, jitterRate=-0.0566687136888504}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T03:00:53,487 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fbf7ef74de7c98db6b37604b08e67282 2024-11-20T03:00:53,487 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1006): Region open journal for fbf7ef74de7c98db6b37604b08e67282: Running coprocessor pre-open hook at 1732071653461Writing region info on filesystem at 1732071653461Initializing all the Stores at 1732071653463 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732071653463Cleaning up temporary data from old regions at 1732071653477 (+14 ms)Running coprocessor post-open hooks at 1732071653487 (+10 ms)Region opened successfully at 1732071653487 2024-11-20T03:00:53,487 INFO [StoreOpener-fc28de84807b58cb0d6ceaa73127debb-1 {}] regionserver.HStore(327): Store=fc28de84807b58cb0d6ceaa73127debb/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T03:00:53,488 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1038): replaying wal for fc28de84807b58cb0d6ceaa73127debb 2024-11-20T03:00:53,488 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2236): Post open deploy tasks for test,nnn,1732071652347.fbf7ef74de7c98db6b37604b08e67282., pid=43, masterSystemTime=1732071653423 2024-11-20T03:00:53,491 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=24 2024-11-20T03:00:53,491 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=24, state=SUCCESS, hasLock=false; OpenRegionProcedure 8a818d7f8283ee7f2b5fae844b64a8ee, server=1c82b83631a2,43333,1732071649765 in 194 msec 2024-11-20T03:00:53,492 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/default/test/fc28de84807b58cb0d6ceaa73127debb 2024-11-20T03:00:53,493 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2266): Finished post open deploy task for test,nnn,1732071652347.fbf7ef74de7c98db6b37604b08e67282. 2024-11-20T03:00:53,493 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(153): Opened test,nnn,1732071652347.fbf7ef74de7c98db6b37604b08e67282. 2024-11-20T03:00:53,493 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/default/test/fc28de84807b58cb0d6ceaa73127debb 2024-11-20T03:00:53,493 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open test,xxx,1732071652347.e8e6d28d8e1806fa7d958327b87df211. 2024-11-20T03:00:53,493 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => e8e6d28d8e1806fa7d958327b87df211, NAME => 'test,xxx,1732071652347.e8e6d28d8e1806fa7d958327b87df211.', STARTKEY => 'xxx', ENDKEY => 'yyy'} 2024-11-20T03:00:53,494 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=fbf7ef74de7c98db6b37604b08e67282, regionState=OPEN, openSeqNum=2, regionLocation=1c82b83631a2,42975,1732071649704 2024-11-20T03:00:53,494 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test e8e6d28d8e1806fa7d958327b87df211 2024-11-20T03:00:53,494 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1048): stopping wal replay for fc28de84807b58cb0d6ceaa73127debb 2024-11-20T03:00:53,494 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated test,xxx,1732071652347.e8e6d28d8e1806fa7d958327b87df211.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T03:00:53,494 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1060): Cleaning up temporary data for fc28de84807b58cb0d6ceaa73127debb 2024-11-20T03:00:53,494 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for e8e6d28d8e1806fa7d958327b87df211 2024-11-20T03:00:53,494 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for e8e6d28d8e1806fa7d958327b87df211 2024-11-20T03:00:53,497 INFO [StoreOpener-e8e6d28d8e1806fa7d958327b87df211-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region e8e6d28d8e1806fa7d958327b87df211 2024-11-20T03:00:53,498 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=43, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure fbf7ef74de7c98db6b37604b08e67282, server=1c82b83631a2,42975,1732071649704 because future has completed 2024-11-20T03:00:53,499 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1093): writing seq id for fc28de84807b58cb0d6ceaa73127debb 2024-11-20T03:00:53,500 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=8a818d7f8283ee7f2b5fae844b64a8ee, ASSIGN in 421 msec 2024-11-20T03:00:53,500 INFO [StoreOpener-e8e6d28d8e1806fa7d958327b87df211-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e8e6d28d8e1806fa7d958327b87df211 columnFamilyName family 2024-11-20T03:00:53,500 DEBUG [StoreOpener-e8e6d28d8e1806fa7d958327b87df211-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T03:00:53,501 INFO [StoreOpener-e8e6d28d8e1806fa7d958327b87df211-1 {}] regionserver.HStore(327): Store=e8e6d28d8e1806fa7d958327b87df211/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T03:00:53,501 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for e8e6d28d8e1806fa7d958327b87df211 2024-11-20T03:00:53,503 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/default/test/e8e6d28d8e1806fa7d958327b87df211 2024-11-20T03:00:53,504 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/default/test/e8e6d28d8e1806fa7d958327b87df211 2024-11-20T03:00:53,504 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/default/test/fc28de84807b58cb0d6ceaa73127debb/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T03:00:53,505 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for e8e6d28d8e1806fa7d958327b87df211 2024-11-20T03:00:53,505 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for e8e6d28d8e1806fa7d958327b87df211 2024-11-20T03:00:53,505 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1114): Opened fc28de84807b58cb0d6ceaa73127debb; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70340665, jitterRate=0.04815758764743805}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T03:00:53,505 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fc28de84807b58cb0d6ceaa73127debb 2024-11-20T03:00:53,506 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1006): Region open journal for fc28de84807b58cb0d6ceaa73127debb: Running coprocessor pre-open hook at 1732071653476Writing region info on filesystem at 1732071653476Initializing all the Stores at 1732071653478 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732071653478Cleaning up temporary data from old regions at 1732071653494 (+16 ms)Running coprocessor post-open hooks at 1732071653506 (+12 ms)Region opened successfully at 1732071653506 2024-11-20T03:00:53,507 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=18 2024-11-20T03:00:53,507 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=18, state=SUCCESS, hasLock=false; OpenRegionProcedure fbf7ef74de7c98db6b37604b08e67282, server=1c82b83631a2,42975,1732071649704 in 170 msec 2024-11-20T03:00:53,507 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2236): Post open deploy tasks for test,fff,1732071652347.fc28de84807b58cb0d6ceaa73127debb., pid=35, masterSystemTime=1732071653406 2024-11-20T03:00:53,508 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for e8e6d28d8e1806fa7d958327b87df211 2024-11-20T03:00:53,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39617 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T03:00:53,510 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2266): Finished post open deploy task for test,fff,1732071652347.fc28de84807b58cb0d6ceaa73127debb. 2024-11-20T03:00:53,511 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(153): Opened test,fff,1732071652347.fc28de84807b58cb0d6ceaa73127debb. 2024-11-20T03:00:53,511 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(132): Open test,ooo,1732071652347.4327d986b4dd716142908b96606b0ad7. 2024-11-20T03:00:53,511 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7752): Opening region: {ENCODED => 4327d986b4dd716142908b96606b0ad7, NAME => 'test,ooo,1732071652347.4327d986b4dd716142908b96606b0ad7.', STARTKEY => 'ooo', ENDKEY => 'ppp'} 2024-11-20T03:00:53,511 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=fbf7ef74de7c98db6b37604b08e67282, ASSIGN in 431 msec 2024-11-20T03:00:53,511 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 4327d986b4dd716142908b96606b0ad7 2024-11-20T03:00:53,511 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(898): Instantiated test,ooo,1732071652347.4327d986b4dd716142908b96606b0ad7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T03:00:53,511 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7794): checking encryption for 4327d986b4dd716142908b96606b0ad7 2024-11-20T03:00:53,512 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7797): checking classloading for 4327d986b4dd716142908b96606b0ad7 2024-11-20T03:00:53,512 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/default/test/e8e6d28d8e1806fa7d958327b87df211/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T03:00:53,514 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=fc28de84807b58cb0d6ceaa73127debb, regionState=OPEN, openSeqNum=2, regionLocation=1c82b83631a2,43333,1732071649765 2024-11-20T03:00:53,515 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened e8e6d28d8e1806fa7d958327b87df211; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69523392, jitterRate=0.035979270935058594}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T03:00:53,515 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e8e6d28d8e1806fa7d958327b87df211 2024-11-20T03:00:53,515 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for e8e6d28d8e1806fa7d958327b87df211: Running coprocessor pre-open hook at 1732071653494Writing region info on filesystem at 1732071653494Initializing all the Stores at 1732071653497 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732071653497Cleaning up temporary data from old regions at 1732071653505 (+8 ms)Running coprocessor post-open hooks at 1732071653515 (+10 ms)Region opened successfully at 1732071653515 2024-11-20T03:00:53,518 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for test,xxx,1732071652347.e8e6d28d8e1806fa7d958327b87df211., pid=33, masterSystemTime=1732071653423 2024-11-20T03:00:53,519 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure fc28de84807b58cb0d6ceaa73127debb, server=1c82b83631a2,43333,1732071649765 because future has completed 2024-11-20T03:00:53,521 INFO [StoreOpener-4327d986b4dd716142908b96606b0ad7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 4327d986b4dd716142908b96606b0ad7 2024-11-20T03:00:53,522 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39617 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=1c82b83631a2,43333,1732071649765, table=test, region=fc28de84807b58cb0d6ceaa73127debb. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-20T03:00:53,524 INFO [StoreOpener-4327d986b4dd716142908b96606b0ad7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4327d986b4dd716142908b96606b0ad7 columnFamilyName family 2024-11-20T03:00:53,524 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-20T03:00:53,524 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for test,xxx,1732071652347.e8e6d28d8e1806fa7d958327b87df211. 2024-11-20T03:00:53,524 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened test,xxx,1732071652347.e8e6d28d8e1806fa7d958327b87df211. 2024-11-20T03:00:53,524 DEBUG [StoreOpener-4327d986b4dd716142908b96606b0ad7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T03:00:53,524 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(132): Open test,ggg,1732071652347.3c5e70743a336dce38e44ee0f8059378. 2024-11-20T03:00:53,526 INFO [StoreOpener-4327d986b4dd716142908b96606b0ad7-1 {}] regionserver.HStore(327): Store=4327d986b4dd716142908b96606b0ad7/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T03:00:53,526 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7752): Opening region: {ENCODED => 3c5e70743a336dce38e44ee0f8059378, NAME => 'test,ggg,1732071652347.3c5e70743a336dce38e44ee0f8059378.', STARTKEY => 'ggg', ENDKEY => 'hhh'} 2024-11-20T03:00:53,526 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1038): replaying wal for 4327d986b4dd716142908b96606b0ad7 2024-11-20T03:00:53,526 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55749, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-20T03:00:53,527 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 3c5e70743a336dce38e44ee0f8059378 2024-11-20T03:00:53,527 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(898): Instantiated test,ggg,1732071652347.3c5e70743a336dce38e44ee0f8059378.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T03:00:53,527 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7794): checking encryption for 3c5e70743a336dce38e44ee0f8059378 2024-11-20T03:00:53,527 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7797): checking classloading for 3c5e70743a336dce38e44ee0f8059378 2024-11-20T03:00:53,528 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/default/test/4327d986b4dd716142908b96606b0ad7 2024-11-20T03:00:53,529 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=10 2024-11-20T03:00:53,529 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure fc28de84807b58cb0d6ceaa73127debb, server=1c82b83631a2,43333,1732071649765 in 245 msec 2024-11-20T03:00:53,530 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/default/test/4327d986b4dd716142908b96606b0ad7 2024-11-20T03:00:53,531 INFO [StoreOpener-3c5e70743a336dce38e44ee0f8059378-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 3c5e70743a336dce38e44ee0f8059378 2024-11-20T03:00:53,531 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1048): stopping wal replay for 4327d986b4dd716142908b96606b0ad7 2024-11-20T03:00:53,532 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1060): Cleaning up temporary data for 4327d986b4dd716142908b96606b0ad7 2024-11-20T03:00:53,533 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(132): Open test,sss,1732071652347.550bbfe722915166ce74fec36828c508. 2024-11-20T03:00:53,533 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=fc28de84807b58cb0d6ceaa73127debb, ASSIGN in 453 msec 2024-11-20T03:00:53,533 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7752): Opening region: {ENCODED => 550bbfe722915166ce74fec36828c508, NAME => 'test,sss,1732071652347.550bbfe722915166ce74fec36828c508.', STARTKEY => 'sss', ENDKEY => 'ttt'} 2024-11-20T03:00:53,533 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 550bbfe722915166ce74fec36828c508 2024-11-20T03:00:53,533 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(898): Instantiated test,sss,1732071652347.550bbfe722915166ce74fec36828c508.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T03:00:53,534 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7794): checking encryption for 550bbfe722915166ce74fec36828c508 2024-11-20T03:00:53,534 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7797): checking classloading for 550bbfe722915166ce74fec36828c508 2024-11-20T03:00:53,535 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=e8e6d28d8e1806fa7d958327b87df211, regionState=OPEN, openSeqNum=2, regionLocation=1c82b83631a2,42975,1732071649704 2024-11-20T03:00:53,535 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1093): writing seq id for 4327d986b4dd716142908b96606b0ad7 2024-11-20T03:00:53,535 INFO [StoreOpener-3c5e70743a336dce38e44ee0f8059378-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3c5e70743a336dce38e44ee0f8059378 columnFamilyName family 2024-11-20T03:00:53,536 DEBUG [StoreOpener-3c5e70743a336dce38e44ee0f8059378-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T03:00:53,538 INFO [StoreOpener-3c5e70743a336dce38e44ee0f8059378-1 {}] regionserver.HStore(327): Store=3c5e70743a336dce38e44ee0f8059378/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T03:00:53,538 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1038): replaying wal for 3c5e70743a336dce38e44ee0f8059378 2024-11-20T03:00:53,541 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/default/test/3c5e70743a336dce38e44ee0f8059378 2024-11-20T03:00:53,542 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/default/test/3c5e70743a336dce38e44ee0f8059378 2024-11-20T03:00:53,542 INFO [StoreOpener-550bbfe722915166ce74fec36828c508-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 550bbfe722915166ce74fec36828c508 2024-11-20T03:00:53,542 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure e8e6d28d8e1806fa7d958327b87df211, server=1c82b83631a2,42975,1732071649704 because future has completed 2024-11-20T03:00:53,543 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1048): stopping wal replay for 3c5e70743a336dce38e44ee0f8059378 2024-11-20T03:00:53,543 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1060): Cleaning up temporary data for 3c5e70743a336dce38e44ee0f8059378 2024-11-20T03:00:53,558 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1093): writing seq id for 3c5e70743a336dce38e44ee0f8059378 2024-11-20T03:00:53,559 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/default/test/4327d986b4dd716142908b96606b0ad7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T03:00:53,561 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1114): Opened 4327d986b4dd716142908b96606b0ad7; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69221872, jitterRate=0.03148627281188965}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T03:00:53,561 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4327d986b4dd716142908b96606b0ad7 2024-11-20T03:00:53,562 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1006): Region open journal for 4327d986b4dd716142908b96606b0ad7: Running coprocessor pre-open hook at 1732071653512Writing region info on filesystem at 1732071653512Initializing all the Stores at 1732071653516 (+4 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732071653516Cleaning up temporary data from old regions at 1732071653532 (+16 ms)Running coprocessor post-open hooks at 1732071653561 (+29 ms)Region opened successfully at 1732071653562 (+1 ms) 2024-11-20T03:00:53,562 INFO [StoreOpener-550bbfe722915166ce74fec36828c508-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 550bbfe722915166ce74fec36828c508 columnFamilyName family 2024-11-20T03:00:53,562 DEBUG [StoreOpener-550bbfe722915166ce74fec36828c508-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T03:00:53,562 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/default/test/3c5e70743a336dce38e44ee0f8059378/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T03:00:53,563 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=28 2024-11-20T03:00:53,563 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=28, state=SUCCESS, hasLock=false; OpenRegionProcedure e8e6d28d8e1806fa7d958327b87df211, server=1c82b83631a2,42975,1732071649704 in 280 msec 2024-11-20T03:00:53,563 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1114): Opened 3c5e70743a336dce38e44ee0f8059378; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70252101, jitterRate=0.046837881207466125}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T03:00:53,563 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3c5e70743a336dce38e44ee0f8059378 2024-11-20T03:00:53,563 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1006): Region open journal for 3c5e70743a336dce38e44ee0f8059378: Running coprocessor pre-open hook at 1732071653527Writing region info on filesystem at 1732071653527Initializing all the Stores at 1732071653530 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732071653530Cleaning up temporary data from old regions at 1732071653543 (+13 ms)Running coprocessor post-open hooks at 1732071653563 (+20 ms)Region opened successfully at 1732071653563 2024-11-20T03:00:53,564 INFO [StoreOpener-550bbfe722915166ce74fec36828c508-1 {}] regionserver.HStore(327): Store=550bbfe722915166ce74fec36828c508/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T03:00:53,564 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1038): replaying wal for 550bbfe722915166ce74fec36828c508 2024-11-20T03:00:53,565 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ooo,1732071652347.4327d986b4dd716142908b96606b0ad7., pid=37, masterSystemTime=1732071653406 2024-11-20T03:00:53,565 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ggg,1732071652347.3c5e70743a336dce38e44ee0f8059378., pid=42, masterSystemTime=1732071653423 2024-11-20T03:00:53,566 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/default/test/550bbfe722915166ce74fec36828c508 2024-11-20T03:00:53,567 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=e8e6d28d8e1806fa7d958327b87df211, ASSIGN in 487 msec 2024-11-20T03:00:53,568 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/default/test/550bbfe722915166ce74fec36828c508 2024-11-20T03:00:53,569 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ooo,1732071652347.4327d986b4dd716142908b96606b0ad7. 2024-11-20T03:00:53,569 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(153): Opened test,ooo,1732071652347.4327d986b4dd716142908b96606b0ad7. 2024-11-20T03:00:53,569 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(132): Open test,ccc,1732071652347.ccd7aea6ec14848f8c6730c88ce1c780. 2024-11-20T03:00:53,569 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7752): Opening region: {ENCODED => ccd7aea6ec14848f8c6730c88ce1c780, NAME => 'test,ccc,1732071652347.ccd7aea6ec14848f8c6730c88ce1c780.', STARTKEY => 'ccc', ENDKEY => 'ddd'} 2024-11-20T03:00:53,570 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test ccd7aea6ec14848f8c6730c88ce1c780 2024-11-20T03:00:53,570 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(898): Instantiated test,ccc,1732071652347.ccd7aea6ec14848f8c6730c88ce1c780.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T03:00:53,570 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7794): checking encryption for ccd7aea6ec14848f8c6730c88ce1c780 2024-11-20T03:00:53,570 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7797): checking classloading for ccd7aea6ec14848f8c6730c88ce1c780 2024-11-20T03:00:53,571 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1048): stopping wal replay for 550bbfe722915166ce74fec36828c508 2024-11-20T03:00:53,571 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1060): Cleaning up temporary data for 550bbfe722915166ce74fec36828c508 2024-11-20T03:00:53,571 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=4327d986b4dd716142908b96606b0ad7, regionState=OPEN, openSeqNum=2, regionLocation=1c82b83631a2,43333,1732071649765 2024-11-20T03:00:53,572 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ggg,1732071652347.3c5e70743a336dce38e44ee0f8059378. 2024-11-20T03:00:53,572 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(153): Opened test,ggg,1732071652347.3c5e70743a336dce38e44ee0f8059378. 2024-11-20T03:00:53,572 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(132): Open test,ppp,1732071652347.ef70776250de5f4e8ad7bd24027abe63. 2024-11-20T03:00:53,572 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7752): Opening region: {ENCODED => ef70776250de5f4e8ad7bd24027abe63, NAME => 'test,ppp,1732071652347.ef70776250de5f4e8ad7bd24027abe63.', STARTKEY => 'ppp', ENDKEY => 'qqq'} 2024-11-20T03:00:53,573 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test ef70776250de5f4e8ad7bd24027abe63 2024-11-20T03:00:53,573 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(898): Instantiated test,ppp,1732071652347.ef70776250de5f4e8ad7bd24027abe63.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T03:00:53,573 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7794): checking encryption for ef70776250de5f4e8ad7bd24027abe63 2024-11-20T03:00:53,573 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7797): checking classloading for ef70776250de5f4e8ad7bd24027abe63 2024-11-20T03:00:53,575 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1093): writing seq id for 550bbfe722915166ce74fec36828c508 2024-11-20T03:00:53,577 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4327d986b4dd716142908b96606b0ad7, server=1c82b83631a2,43333,1732071649765 because future has completed 2024-11-20T03:00:53,579 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=3c5e70743a336dce38e44ee0f8059378, regionState=OPEN, openSeqNum=2, regionLocation=1c82b83631a2,42975,1732071649704 2024-11-20T03:00:53,583 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=42, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3c5e70743a336dce38e44ee0f8059378, server=1c82b83631a2,42975,1732071649704 because future has completed 2024-11-20T03:00:53,587 INFO [StoreOpener-ccd7aea6ec14848f8c6730c88ce1c780-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region ccd7aea6ec14848f8c6730c88ce1c780 2024-11-20T03:00:53,590 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=19 2024-11-20T03:00:53,590 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=19, state=SUCCESS, hasLock=false; OpenRegionProcedure 4327d986b4dd716142908b96606b0ad7, server=1c82b83631a2,43333,1732071649765 in 298 msec 2024-11-20T03:00:53,590 INFO [StoreOpener-ccd7aea6ec14848f8c6730c88ce1c780-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ccd7aea6ec14848f8c6730c88ce1c780 columnFamilyName family 2024-11-20T03:00:53,590 DEBUG [StoreOpener-ccd7aea6ec14848f8c6730c88ce1c780-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T03:00:53,591 INFO [StoreOpener-ccd7aea6ec14848f8c6730c88ce1c780-1 {}] regionserver.HStore(327): Store=ccd7aea6ec14848f8c6730c88ce1c780/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T03:00:53,592 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1038): replaying wal for ccd7aea6ec14848f8c6730c88ce1c780 2024-11-20T03:00:53,593 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/default/test/ccd7aea6ec14848f8c6730c88ce1c780 2024-11-20T03:00:53,594 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/default/test/ccd7aea6ec14848f8c6730c88ce1c780 2024-11-20T03:00:53,595 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=4327d986b4dd716142908b96606b0ad7, ASSIGN in 514 msec 2024-11-20T03:00:53,597 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1048): stopping wal replay for ccd7aea6ec14848f8c6730c88ce1c780 2024-11-20T03:00:53,597 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1060): Cleaning up temporary data for ccd7aea6ec14848f8c6730c88ce1c780 2024-11-20T03:00:53,601 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1093): writing seq id for ccd7aea6ec14848f8c6730c88ce1c780 2024-11-20T03:00:53,603 INFO [StoreOpener-ef70776250de5f4e8ad7bd24027abe63-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region ef70776250de5f4e8ad7bd24027abe63 2024-11-20T03:00:53,606 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=42, resume processing ppid=11 2024-11-20T03:00:53,606 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure 3c5e70743a336dce38e44ee0f8059378, server=1c82b83631a2,42975,1732071649704 in 274 msec 2024-11-20T03:00:53,607 INFO [StoreOpener-ef70776250de5f4e8ad7bd24027abe63-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ef70776250de5f4e8ad7bd24027abe63 columnFamilyName family 2024-11-20T03:00:53,607 DEBUG [StoreOpener-ef70776250de5f4e8ad7bd24027abe63-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T03:00:53,609 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=3c5e70743a336dce38e44ee0f8059378, ASSIGN in 530 msec 2024-11-20T03:00:53,609 INFO [StoreOpener-ef70776250de5f4e8ad7bd24027abe63-1 {}] regionserver.HStore(327): Store=ef70776250de5f4e8ad7bd24027abe63/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T03:00:53,609 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1038): replaying wal for ef70776250de5f4e8ad7bd24027abe63 2024-11-20T03:00:53,610 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/default/test/ef70776250de5f4e8ad7bd24027abe63 2024-11-20T03:00:53,611 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/default/test/ef70776250de5f4e8ad7bd24027abe63 2024-11-20T03:00:53,611 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/default/test/550bbfe722915166ce74fec36828c508/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T03:00:53,612 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1048): stopping wal replay for ef70776250de5f4e8ad7bd24027abe63 2024-11-20T03:00:53,612 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1114): Opened 550bbfe722915166ce74fec36828c508; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62509305, jitterRate=-0.06853877007961273}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T03:00:53,612 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1060): Cleaning up temporary data for ef70776250de5f4e8ad7bd24027abe63 2024-11-20T03:00:53,612 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 550bbfe722915166ce74fec36828c508 2024-11-20T03:00:53,612 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1006): Region open journal for 550bbfe722915166ce74fec36828c508: Running coprocessor pre-open hook at 1732071653534Writing region info on filesystem at 1732071653534Initializing all the Stores at 1732071653535 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732071653535Cleaning up temporary data from old regions at 1732071653571 (+36 ms)Running coprocessor post-open hooks at 1732071653612 (+41 ms)Region opened successfully at 1732071653612 2024-11-20T03:00:53,613 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2236): Post open deploy tasks for test,sss,1732071652347.550bbfe722915166ce74fec36828c508., pid=47, masterSystemTime=1732071653523 2024-11-20T03:00:53,615 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1093): writing seq id for ef70776250de5f4e8ad7bd24027abe63 2024-11-20T03:00:53,615 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/default/test/ccd7aea6ec14848f8c6730c88ce1c780/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T03:00:53,616 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1114): Opened ccd7aea6ec14848f8c6730c88ce1c780; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72735046, jitterRate=0.0838366448879242}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T03:00:53,616 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ccd7aea6ec14848f8c6730c88ce1c780 2024-11-20T03:00:53,617 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2266): Finished post open deploy task for test,sss,1732071652347.550bbfe722915166ce74fec36828c508. 2024-11-20T03:00:53,617 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1006): Region open journal for ccd7aea6ec14848f8c6730c88ce1c780: Running coprocessor pre-open hook at 1732071653570Writing region info on filesystem at 1732071653570Initializing all the Stores at 1732071653572 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732071653572Cleaning up temporary data from old regions at 1732071653597 (+25 ms)Running coprocessor post-open hooks at 1732071653617 (+20 ms)Region opened successfully at 1732071653617 2024-11-20T03:00:53,617 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(153): Opened test,sss,1732071652347.550bbfe722915166ce74fec36828c508. 2024-11-20T03:00:53,617 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(132): Open test,eee,1732071652347.2fe87cdaf67461ff5c508d851e418c66. 2024-11-20T03:00:53,617 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7752): Opening region: {ENCODED => 2fe87cdaf67461ff5c508d851e418c66, NAME => 'test,eee,1732071652347.2fe87cdaf67461ff5c508d851e418c66.', STARTKEY => 'eee', ENDKEY => 'fff'} 2024-11-20T03:00:53,617 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 2fe87cdaf67461ff5c508d851e418c66 2024-11-20T03:00:53,618 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(898): Instantiated test,eee,1732071652347.2fe87cdaf67461ff5c508d851e418c66.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T03:00:53,618 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7794): checking encryption for 2fe87cdaf67461ff5c508d851e418c66 2024-11-20T03:00:53,618 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7797): checking classloading for 2fe87cdaf67461ff5c508d851e418c66 2024-11-20T03:00:53,619 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ccc,1732071652347.ccd7aea6ec14848f8c6730c88ce1c780., pid=34, masterSystemTime=1732071653406 2024-11-20T03:00:53,620 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=550bbfe722915166ce74fec36828c508, regionState=OPEN, openSeqNum=2, regionLocation=1c82b83631a2,36133,1732071649570 2024-11-20T03:00:53,620 INFO [StoreOpener-2fe87cdaf67461ff5c508d851e418c66-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 2fe87cdaf67461ff5c508d851e418c66 2024-11-20T03:00:53,621 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/default/test/ef70776250de5f4e8ad7bd24027abe63/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T03:00:53,622 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1114): Opened ef70776250de5f4e8ad7bd24027abe63; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68358201, jitterRate=0.01861657202243805}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T03:00:53,622 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ef70776250de5f4e8ad7bd24027abe63 2024-11-20T03:00:53,623 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1006): Region open journal for ef70776250de5f4e8ad7bd24027abe63: Running coprocessor pre-open hook at 1732071653573Writing region info on filesystem at 1732071653573Initializing all the Stores at 1732071653575 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732071653575Cleaning up temporary data from old regions at 1732071653612 (+37 ms)Running coprocessor post-open hooks at 1732071653622 (+10 ms)Region opened successfully at 1732071653622 2024-11-20T03:00:53,623 INFO [StoreOpener-2fe87cdaf67461ff5c508d851e418c66-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2fe87cdaf67461ff5c508d851e418c66 columnFamilyName family 2024-11-20T03:00:53,623 DEBUG [StoreOpener-2fe87cdaf67461ff5c508d851e418c66-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T03:00:53,624 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ccc,1732071652347.ccd7aea6ec14848f8c6730c88ce1c780. 2024-11-20T03:00:53,624 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(153): Opened test,ccc,1732071652347.ccd7aea6ec14848f8c6730c88ce1c780. 2024-11-20T03:00:53,624 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=47, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 550bbfe722915166ce74fec36828c508, server=1c82b83631a2,36133,1732071649570 because future has completed 2024-11-20T03:00:53,624 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(132): Open test,www,1732071652347.8a06021a0c54a875234d29e87253d48a. 2024-11-20T03:00:53,624 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7752): Opening region: {ENCODED => 8a06021a0c54a875234d29e87253d48a, NAME => 'test,www,1732071652347.8a06021a0c54a875234d29e87253d48a.', STARTKEY => 'www', ENDKEY => 'xxx'} 2024-11-20T03:00:53,624 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 8a06021a0c54a875234d29e87253d48a 2024-11-20T03:00:53,625 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(898): Instantiated test,www,1732071652347.8a06021a0c54a875234d29e87253d48a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T03:00:53,625 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7794): checking encryption for 8a06021a0c54a875234d29e87253d48a 2024-11-20T03:00:53,625 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7797): checking classloading for 8a06021a0c54a875234d29e87253d48a 2024-11-20T03:00:53,625 INFO [StoreOpener-2fe87cdaf67461ff5c508d851e418c66-1 {}] regionserver.HStore(327): Store=2fe87cdaf67461ff5c508d851e418c66/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T03:00:53,625 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1038): replaying wal for 2fe87cdaf67461ff5c508d851e418c66 2024-11-20T03:00:53,625 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ppp,1732071652347.ef70776250de5f4e8ad7bd24027abe63., pid=44, masterSystemTime=1732071653423 2024-11-20T03:00:53,627 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=ccd7aea6ec14848f8c6730c88ce1c780, regionState=OPEN, openSeqNum=2, regionLocation=1c82b83631a2,43333,1732071649765 2024-11-20T03:00:53,627 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/default/test/2fe87cdaf67461ff5c508d851e418c66 2024-11-20T03:00:53,627 INFO [StoreOpener-8a06021a0c54a875234d29e87253d48a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 8a06021a0c54a875234d29e87253d48a 2024-11-20T03:00:53,628 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/default/test/2fe87cdaf67461ff5c508d851e418c66 2024-11-20T03:00:53,629 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ppp,1732071652347.ef70776250de5f4e8ad7bd24027abe63. 2024-11-20T03:00:53,629 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(153): Opened test,ppp,1732071652347.ef70776250de5f4e8ad7bd24027abe63. 2024-11-20T03:00:53,629 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(132): Open test,qqq,1732071652347.30cc99ff17ee8d21b793ce798b4998ba. 2024-11-20T03:00:53,629 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7752): Opening region: {ENCODED => 30cc99ff17ee8d21b793ce798b4998ba, NAME => 'test,qqq,1732071652347.30cc99ff17ee8d21b793ce798b4998ba.', STARTKEY => 'qqq', ENDKEY => 'rrr'} 2024-11-20T03:00:53,629 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 30cc99ff17ee8d21b793ce798b4998ba 2024-11-20T03:00:53,630 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1048): stopping wal replay for 2fe87cdaf67461ff5c508d851e418c66 2024-11-20T03:00:53,630 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(898): Instantiated test,qqq,1732071652347.30cc99ff17ee8d21b793ce798b4998ba.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T03:00:53,630 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1060): Cleaning up temporary data for 2fe87cdaf67461ff5c508d851e418c66 2024-11-20T03:00:53,630 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7794): checking encryption for 30cc99ff17ee8d21b793ce798b4998ba 2024-11-20T03:00:53,630 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7797): checking classloading for 30cc99ff17ee8d21b793ce798b4998ba 2024-11-20T03:00:53,631 INFO [StoreOpener-8a06021a0c54a875234d29e87253d48a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8a06021a0c54a875234d29e87253d48a columnFamilyName family 2024-11-20T03:00:53,631 DEBUG [StoreOpener-8a06021a0c54a875234d29e87253d48a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T03:00:53,631 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure ccd7aea6ec14848f8c6730c88ce1c780, server=1c82b83631a2,43333,1732071649765 because future has completed 2024-11-20T03:00:53,632 INFO [StoreOpener-8a06021a0c54a875234d29e87253d48a-1 {}] regionserver.HStore(327): Store=8a06021a0c54a875234d29e87253d48a/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T03:00:53,632 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1038): replaying wal for 8a06021a0c54a875234d29e87253d48a 2024-11-20T03:00:53,633 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1093): writing seq id for 2fe87cdaf67461ff5c508d851e418c66 2024-11-20T03:00:53,633 INFO [StoreOpener-30cc99ff17ee8d21b793ce798b4998ba-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 30cc99ff17ee8d21b793ce798b4998ba 2024-11-20T03:00:53,633 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/default/test/8a06021a0c54a875234d29e87253d48a 2024-11-20T03:00:53,634 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/default/test/8a06021a0c54a875234d29e87253d48a 2024-11-20T03:00:53,635 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1048): stopping wal replay for 8a06021a0c54a875234d29e87253d48a 2024-11-20T03:00:53,635 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1060): Cleaning up temporary data for 8a06021a0c54a875234d29e87253d48a 2024-11-20T03:00:53,636 INFO [StoreOpener-30cc99ff17ee8d21b793ce798b4998ba-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 30cc99ff17ee8d21b793ce798b4998ba columnFamilyName family 2024-11-20T03:00:53,636 DEBUG [StoreOpener-30cc99ff17ee8d21b793ce798b4998ba-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T03:00:53,637 INFO [StoreOpener-30cc99ff17ee8d21b793ce798b4998ba-1 {}] regionserver.HStore(327): Store=30cc99ff17ee8d21b793ce798b4998ba/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T03:00:53,637 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1038): replaying wal for 30cc99ff17ee8d21b793ce798b4998ba 2024-11-20T03:00:53,638 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/default/test/30cc99ff17ee8d21b793ce798b4998ba 2024-11-20T03:00:53,639 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1093): writing seq id for 8a06021a0c54a875234d29e87253d48a 2024-11-20T03:00:53,639 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/default/test/30cc99ff17ee8d21b793ce798b4998ba 2024-11-20T03:00:53,640 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1048): stopping wal replay for 30cc99ff17ee8d21b793ce798b4998ba 2024-11-20T03:00:53,640 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1060): Cleaning up temporary data for 30cc99ff17ee8d21b793ce798b4998ba 2024-11-20T03:00:53,640 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=7 2024-11-20T03:00:53,640 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure ccd7aea6ec14848f8c6730c88ce1c780, server=1c82b83631a2,43333,1732071649765 in 362 msec 2024-11-20T03:00:53,643 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1093): writing seq id for 30cc99ff17ee8d21b793ce798b4998ba 2024-11-20T03:00:53,643 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/default/test/2fe87cdaf67461ff5c508d851e418c66/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T03:00:53,644 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=ccd7aea6ec14848f8c6730c88ce1c780, ASSIGN in 564 msec 2024-11-20T03:00:53,644 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1114): Opened 2fe87cdaf67461ff5c508d851e418c66; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71000391, jitterRate=0.05798827111721039}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T03:00:53,644 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2fe87cdaf67461ff5c508d851e418c66 2024-11-20T03:00:53,644 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1006): Region open journal for 2fe87cdaf67461ff5c508d851e418c66: Running coprocessor pre-open hook at 1732071653618Writing region info on filesystem at 1732071653618Initializing all the Stores at 1732071653619 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732071653620 (+1 ms)Cleaning up temporary data from old regions at 1732071653630 (+10 ms)Running coprocessor post-open hooks at 1732071653644 (+14 ms)Region opened successfully at 1732071653644 2024-11-20T03:00:53,646 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2236): Post open deploy tasks for test,eee,1732071652347.2fe87cdaf67461ff5c508d851e418c66., pid=52, masterSystemTime=1732071653523 2024-11-20T03:00:53,646 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=ef70776250de5f4e8ad7bd24027abe63, regionState=OPEN, openSeqNum=2, regionLocation=1c82b83631a2,42975,1732071649704 2024-11-20T03:00:53,646 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/default/test/8a06021a0c54a875234d29e87253d48a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T03:00:53,647 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1114): Opened 8a06021a0c54a875234d29e87253d48a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68506347, jitterRate=0.020824119448661804}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T03:00:53,647 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8a06021a0c54a875234d29e87253d48a 2024-11-20T03:00:53,647 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1006): Region open journal for 8a06021a0c54a875234d29e87253d48a: Running coprocessor pre-open hook at 1732071653625Writing region info on filesystem at 1732071653625Initializing all the Stores at 1732071653627 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732071653627Cleaning up temporary data from old regions at 1732071653635 (+8 ms)Running coprocessor post-open hooks at 1732071653647 (+12 ms)Region opened successfully at 1732071653647 2024-11-20T03:00:53,649 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2236): Post open deploy tasks for test,www,1732071652347.8a06021a0c54a875234d29e87253d48a., pid=39, masterSystemTime=1732071653406 2024-11-20T03:00:53,649 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2266): Finished post open deploy task for test,eee,1732071652347.2fe87cdaf67461ff5c508d851e418c66. 2024-11-20T03:00:53,649 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(153): Opened test,eee,1732071652347.2fe87cdaf67461ff5c508d851e418c66. 2024-11-20T03:00:53,649 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=44, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure ef70776250de5f4e8ad7bd24027abe63, server=1c82b83631a2,42975,1732071649704 because future has completed 2024-11-20T03:00:53,649 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(132): Open test,vvv,1732071652347.45e8fc1ea8df34f038aeb00318494097. 2024-11-20T03:00:53,650 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7752): Opening region: {ENCODED => 45e8fc1ea8df34f038aeb00318494097, NAME => 'test,vvv,1732071652347.45e8fc1ea8df34f038aeb00318494097.', STARTKEY => 'vvv', ENDKEY => 'www'} 2024-11-20T03:00:53,650 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 45e8fc1ea8df34f038aeb00318494097 2024-11-20T03:00:53,650 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(898): Instantiated test,vvv,1732071652347.45e8fc1ea8df34f038aeb00318494097.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T03:00:53,650 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7794): checking encryption for 45e8fc1ea8df34f038aeb00318494097 2024-11-20T03:00:53,650 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7797): checking classloading for 45e8fc1ea8df34f038aeb00318494097 2024-11-20T03:00:53,651 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/default/test/30cc99ff17ee8d21b793ce798b4998ba/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T03:00:53,652 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=2fe87cdaf67461ff5c508d851e418c66, regionState=OPEN, openSeqNum=2, regionLocation=1c82b83631a2,36133,1732071649570 2024-11-20T03:00:53,652 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1114): Opened 30cc99ff17ee8d21b793ce798b4998ba; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71403402, jitterRate=0.06399360299110413}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T03:00:53,653 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 30cc99ff17ee8d21b793ce798b4998ba 2024-11-20T03:00:53,653 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1006): Region open journal for 30cc99ff17ee8d21b793ce798b4998ba: Running coprocessor pre-open hook at 1732071653630Writing region info on filesystem at 1732071653630Initializing all the Stores at 1732071653633 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732071653633Cleaning up temporary data from old regions at 1732071653640 (+7 ms)Running coprocessor post-open hooks at 1732071653653 (+13 ms)Region opened successfully at 1732071653653 2024-11-20T03:00:53,653 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2266): Finished post open deploy task for test,www,1732071652347.8a06021a0c54a875234d29e87253d48a. 2024-11-20T03:00:53,653 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(153): Opened test,www,1732071652347.8a06021a0c54a875234d29e87253d48a. 2024-11-20T03:00:53,653 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(132): Open test,mmm,1732071652347.38fa6330e381ad6ba7391bf9050c289e. 2024-11-20T03:00:53,653 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7752): Opening region: {ENCODED => 38fa6330e381ad6ba7391bf9050c289e, NAME => 'test,mmm,1732071652347.38fa6330e381ad6ba7391bf9050c289e.', STARTKEY => 'mmm', ENDKEY => 'nnn'} 2024-11-20T03:00:53,654 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 38fa6330e381ad6ba7391bf9050c289e 2024-11-20T03:00:53,654 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(898): Instantiated test,mmm,1732071652347.38fa6330e381ad6ba7391bf9050c289e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T03:00:53,654 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7794): checking encryption for 38fa6330e381ad6ba7391bf9050c289e 2024-11-20T03:00:53,654 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7797): checking classloading for 38fa6330e381ad6ba7391bf9050c289e 2024-11-20T03:00:53,655 INFO [StoreOpener-45e8fc1ea8df34f038aeb00318494097-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 45e8fc1ea8df34f038aeb00318494097 2024-11-20T03:00:53,655 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2236): Post open deploy tasks for test,qqq,1732071652347.30cc99ff17ee8d21b793ce798b4998ba., pid=45, masterSystemTime=1732071653423 2024-11-20T03:00:53,656 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=52, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2fe87cdaf67461ff5c508d851e418c66, server=1c82b83631a2,36133,1732071649570 because future has completed 2024-11-20T03:00:53,657 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=8a06021a0c54a875234d29e87253d48a, regionState=OPEN, openSeqNum=2, regionLocation=1c82b83631a2,43333,1732071649765 2024-11-20T03:00:53,658 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2266): Finished post open deploy task for test,qqq,1732071652347.30cc99ff17ee8d21b793ce798b4998ba. 2024-11-20T03:00:53,658 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(153): Opened test,qqq,1732071652347.30cc99ff17ee8d21b793ce798b4998ba. 2024-11-20T03:00:53,658 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(132): Open test,uuu,1732071652347.49f9e405ee146272624d3d205a4ebf9e. 2024-11-20T03:00:53,658 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7752): Opening region: {ENCODED => 49f9e405ee146272624d3d205a4ebf9e, NAME => 'test,uuu,1732071652347.49f9e405ee146272624d3d205a4ebf9e.', STARTKEY => 'uuu', ENDKEY => 'vvv'} 2024-11-20T03:00:53,659 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 49f9e405ee146272624d3d205a4ebf9e 2024-11-20T03:00:53,659 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(898): Instantiated test,uuu,1732071652347.49f9e405ee146272624d3d205a4ebf9e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T03:00:53,659 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7794): checking encryption for 49f9e405ee146272624d3d205a4ebf9e 2024-11-20T03:00:53,659 INFO [StoreOpener-45e8fc1ea8df34f038aeb00318494097-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 45e8fc1ea8df34f038aeb00318494097 columnFamilyName family 2024-11-20T03:00:53,659 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7797): checking classloading for 49f9e405ee146272624d3d205a4ebf9e 2024-11-20T03:00:53,659 DEBUG [StoreOpener-45e8fc1ea8df34f038aeb00318494097-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T03:00:53,660 INFO [StoreOpener-38fa6330e381ad6ba7391bf9050c289e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 38fa6330e381ad6ba7391bf9050c289e 2024-11-20T03:00:53,660 INFO [StoreOpener-45e8fc1ea8df34f038aeb00318494097-1 {}] regionserver.HStore(327): Store=45e8fc1ea8df34f038aeb00318494097/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T03:00:53,661 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8a06021a0c54a875234d29e87253d48a, server=1c82b83631a2,43333,1732071649765 because future has completed 2024-11-20T03:00:53,662 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1038): replaying wal for 45e8fc1ea8df34f038aeb00318494097 2024-11-20T03:00:53,663 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=30cc99ff17ee8d21b793ce798b4998ba, regionState=OPEN, openSeqNum=2, regionLocation=1c82b83631a2,42975,1732071649704 2024-11-20T03:00:53,663 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/default/test/45e8fc1ea8df34f038aeb00318494097 2024-11-20T03:00:53,663 INFO [StoreOpener-38fa6330e381ad6ba7391bf9050c289e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 38fa6330e381ad6ba7391bf9050c289e columnFamilyName family 2024-11-20T03:00:53,663 DEBUG [StoreOpener-38fa6330e381ad6ba7391bf9050c289e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T03:00:53,664 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/default/test/45e8fc1ea8df34f038aeb00318494097 2024-11-20T03:00:53,664 INFO [StoreOpener-49f9e405ee146272624d3d205a4ebf9e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 49f9e405ee146272624d3d205a4ebf9e 2024-11-20T03:00:53,665 INFO [StoreOpener-38fa6330e381ad6ba7391bf9050c289e-1 {}] regionserver.HStore(327): Store=38fa6330e381ad6ba7391bf9050c289e/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T03:00:53,665 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1048): stopping wal replay for 45e8fc1ea8df34f038aeb00318494097 2024-11-20T03:00:53,665 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1060): Cleaning up temporary data for 45e8fc1ea8df34f038aeb00318494097 2024-11-20T03:00:53,665 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1038): replaying wal for 38fa6330e381ad6ba7391bf9050c289e 2024-11-20T03:00:53,666 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=45, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure 30cc99ff17ee8d21b793ce798b4998ba, server=1c82b83631a2,42975,1732071649704 because future has completed 2024-11-20T03:00:53,667 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/default/test/38fa6330e381ad6ba7391bf9050c289e 2024-11-20T03:00:53,668 INFO [StoreOpener-49f9e405ee146272624d3d205a4ebf9e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 49f9e405ee146272624d3d205a4ebf9e columnFamilyName family 2024-11-20T03:00:53,668 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/default/test/38fa6330e381ad6ba7391bf9050c289e 2024-11-20T03:00:53,668 DEBUG [StoreOpener-49f9e405ee146272624d3d205a4ebf9e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T03:00:53,668 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1093): writing seq id for 45e8fc1ea8df34f038aeb00318494097 2024-11-20T03:00:53,668 INFO [StoreOpener-49f9e405ee146272624d3d205a4ebf9e-1 {}] regionserver.HStore(327): Store=49f9e405ee146272624d3d205a4ebf9e/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T03:00:53,668 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1048): stopping wal replay for 38fa6330e381ad6ba7391bf9050c289e 2024-11-20T03:00:53,669 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1060): Cleaning up temporary data for 38fa6330e381ad6ba7391bf9050c289e 2024-11-20T03:00:53,669 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1038): replaying wal for 49f9e405ee146272624d3d205a4ebf9e 2024-11-20T03:00:53,670 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/default/test/49f9e405ee146272624d3d205a4ebf9e 2024-11-20T03:00:53,671 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/default/test/49f9e405ee146272624d3d205a4ebf9e 2024-11-20T03:00:53,671 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/default/test/45e8fc1ea8df34f038aeb00318494097/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T03:00:53,672 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1048): stopping wal replay for 49f9e405ee146272624d3d205a4ebf9e 2024-11-20T03:00:53,672 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1060): Cleaning up temporary data for 49f9e405ee146272624d3d205a4ebf9e 2024-11-20T03:00:53,672 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1093): writing seq id for 38fa6330e381ad6ba7391bf9050c289e 2024-11-20T03:00:53,672 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1114): Opened 45e8fc1ea8df34f038aeb00318494097; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58786590, jitterRate=-0.12401154637336731}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T03:00:53,672 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 45e8fc1ea8df34f038aeb00318494097 2024-11-20T03:00:53,672 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1006): Region open journal for 45e8fc1ea8df34f038aeb00318494097: Running coprocessor pre-open hook at 1732071653650Writing region info on filesystem at 1732071653650Initializing all the Stores at 1732071653653 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732071653653Cleaning up temporary data from old regions at 1732071653665 (+12 ms)Running coprocessor post-open hooks at 1732071653672 (+7 ms)Region opened successfully at 1732071653672 2024-11-20T03:00:53,673 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=27 2024-11-20T03:00:53,673 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=27, state=SUCCESS, hasLock=false; OpenRegionProcedure 8a06021a0c54a875234d29e87253d48a, server=1c82b83631a2,43333,1732071649765 in 371 msec 2024-11-20T03:00:53,674 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2236): Post open deploy tasks for test,vvv,1732071652347.45e8fc1ea8df34f038aeb00318494097., pid=48, masterSystemTime=1732071653523 2024-11-20T03:00:53,677 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=8a06021a0c54a875234d29e87253d48a, ASSIGN in 597 msec 2024-11-20T03:00:53,678 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2266): Finished post open deploy task for test,vvv,1732071652347.45e8fc1ea8df34f038aeb00318494097. 2024-11-20T03:00:53,678 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(153): Opened test,vvv,1732071652347.45e8fc1ea8df34f038aeb00318494097. 2024-11-20T03:00:53,678 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(132): Open test,yyy,1732071652347.295cabe34d2d3dbccccb06cf3aa5bbf6. 2024-11-20T03:00:53,678 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1093): writing seq id for 49f9e405ee146272624d3d205a4ebf9e 2024-11-20T03:00:53,678 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7752): Opening region: {ENCODED => 295cabe34d2d3dbccccb06cf3aa5bbf6, NAME => 'test,yyy,1732071652347.295cabe34d2d3dbccccb06cf3aa5bbf6.', STARTKEY => 'yyy', ENDKEY => 'zzz'} 2024-11-20T03:00:53,678 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/default/test/38fa6330e381ad6ba7391bf9050c289e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T03:00:53,678 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 295cabe34d2d3dbccccb06cf3aa5bbf6 2024-11-20T03:00:53,679 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(898): Instantiated test,yyy,1732071652347.295cabe34d2d3dbccccb06cf3aa5bbf6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T03:00:53,679 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7794): checking encryption for 295cabe34d2d3dbccccb06cf3aa5bbf6 2024-11-20T03:00:53,679 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7797): checking classloading for 295cabe34d2d3dbccccb06cf3aa5bbf6 2024-11-20T03:00:53,679 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1114): Opened 38fa6330e381ad6ba7391bf9050c289e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70370711, jitterRate=0.048605307936668396}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T03:00:53,679 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 38fa6330e381ad6ba7391bf9050c289e 2024-11-20T03:00:53,679 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1006): Region open journal for 38fa6330e381ad6ba7391bf9050c289e: Running coprocessor pre-open hook at 1732071653654Writing region info on filesystem at 1732071653654Initializing all the Stores at 1732071653657 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732071653657Cleaning up temporary data from old regions at 1732071653669 (+12 ms)Running coprocessor post-open hooks at 1732071653679 (+10 ms)Region opened successfully at 1732071653679 2024-11-20T03:00:53,681 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=45e8fc1ea8df34f038aeb00318494097, regionState=OPEN, openSeqNum=2, regionLocation=1c82b83631a2,36133,1732071649570 2024-11-20T03:00:53,681 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2236): Post open deploy tasks for test,mmm,1732071652347.38fa6330e381ad6ba7391bf9050c289e., pid=32, masterSystemTime=1732071653406 2024-11-20T03:00:53,684 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2266): Finished post open deploy task for test,mmm,1732071652347.38fa6330e381ad6ba7391bf9050c289e. 2024-11-20T03:00:53,684 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(153): Opened test,mmm,1732071652347.38fa6330e381ad6ba7391bf9050c289e. 2024-11-20T03:00:53,684 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=48, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 45e8fc1ea8df34f038aeb00318494097, server=1c82b83631a2,36133,1732071649570 because future has completed 2024-11-20T03:00:53,684 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(132): Open test,iii,1732071652347.f928be6f5d7a3631384ebd87c8f80996. 2024-11-20T03:00:53,684 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7752): Opening region: {ENCODED => f928be6f5d7a3631384ebd87c8f80996, NAME => 'test,iii,1732071652347.f928be6f5d7a3631384ebd87c8f80996.', STARTKEY => 'iii', ENDKEY => 'jjj'} 2024-11-20T03:00:53,685 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test f928be6f5d7a3631384ebd87c8f80996 2024-11-20T03:00:53,685 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(898): Instantiated test,iii,1732071652347.f928be6f5d7a3631384ebd87c8f80996.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T03:00:53,685 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7794): checking encryption for f928be6f5d7a3631384ebd87c8f80996 2024-11-20T03:00:53,685 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7797): checking classloading for f928be6f5d7a3631384ebd87c8f80996 2024-11-20T03:00:53,688 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=21 2024-11-20T03:00:53,688 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=21, state=SUCCESS, hasLock=false; OpenRegionProcedure 30cc99ff17ee8d21b793ce798b4998ba, server=1c82b83631a2,42975,1732071649704 in 338 msec 2024-11-20T03:00:53,690 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=30cc99ff17ee8d21b793ce798b4998ba, ASSIGN in 612 msec 2024-11-20T03:00:53,691 INFO [StoreOpener-295cabe34d2d3dbccccb06cf3aa5bbf6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 295cabe34d2d3dbccccb06cf3aa5bbf6 2024-11-20T03:00:53,691 INFO [StoreOpener-f928be6f5d7a3631384ebd87c8f80996-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region f928be6f5d7a3631384ebd87c8f80996 2024-11-20T03:00:53,691 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/default/test/49f9e405ee146272624d3d205a4ebf9e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T03:00:53,693 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1114): Opened 49f9e405ee146272624d3d205a4ebf9e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70521305, jitterRate=0.05084933340549469}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T03:00:53,693 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 49f9e405ee146272624d3d205a4ebf9e 2024-11-20T03:00:53,693 INFO [StoreOpener-295cabe34d2d3dbccccb06cf3aa5bbf6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 295cabe34d2d3dbccccb06cf3aa5bbf6 columnFamilyName family 2024-11-20T03:00:53,693 DEBUG [StoreOpener-295cabe34d2d3dbccccb06cf3aa5bbf6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T03:00:53,693 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1006): Region open journal for 49f9e405ee146272624d3d205a4ebf9e: Running coprocessor pre-open hook at 1732071653660Writing region info on filesystem at 1732071653660Initializing all the Stores at 1732071653662 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732071653662Cleaning up temporary data from old regions at 1732071653672 (+10 ms)Running coprocessor post-open hooks at 1732071653693 (+21 ms)Region opened successfully at 1732071653693 2024-11-20T03:00:53,694 INFO [StoreOpener-295cabe34d2d3dbccccb06cf3aa5bbf6-1 {}] regionserver.HStore(327): Store=295cabe34d2d3dbccccb06cf3aa5bbf6/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T03:00:53,694 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2236): Post open deploy tasks for test,uuu,1732071652347.49f9e405ee146272624d3d205a4ebf9e., pid=40, masterSystemTime=1732071653423 2024-11-20T03:00:53,694 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1038): replaying wal for 295cabe34d2d3dbccccb06cf3aa5bbf6 2024-11-20T03:00:53,694 INFO [StoreOpener-f928be6f5d7a3631384ebd87c8f80996-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f928be6f5d7a3631384ebd87c8f80996 columnFamilyName family 2024-11-20T03:00:53,695 DEBUG [StoreOpener-f928be6f5d7a3631384ebd87c8f80996-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T03:00:53,696 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/default/test/295cabe34d2d3dbccccb06cf3aa5bbf6 2024-11-20T03:00:53,696 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/default/test/295cabe34d2d3dbccccb06cf3aa5bbf6 2024-11-20T03:00:53,696 INFO [StoreOpener-f928be6f5d7a3631384ebd87c8f80996-1 {}] regionserver.HStore(327): Store=f928be6f5d7a3631384ebd87c8f80996/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T03:00:53,697 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1038): replaying wal for f928be6f5d7a3631384ebd87c8f80996 2024-11-20T03:00:53,697 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1048): stopping wal replay for 295cabe34d2d3dbccccb06cf3aa5bbf6 2024-11-20T03:00:53,697 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1060): Cleaning up temporary data for 295cabe34d2d3dbccccb06cf3aa5bbf6 2024-11-20T03:00:53,700 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2266): Finished post open deploy task for test,uuu,1732071652347.49f9e405ee146272624d3d205a4ebf9e. 2024-11-20T03:00:53,700 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39617 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=1c82b83631a2,36133,1732071649570, table=test, region=2fe87cdaf67461ff5c508d851e418c66. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-20T03:00:53,700 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(153): Opened test,uuu,1732071652347.49f9e405ee146272624d3d205a4ebf9e. 2024-11-20T03:00:53,700 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39617 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=1c82b83631a2,36133,1732071649570, table=test, region=550bbfe722915166ce74fec36828c508. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-20T03:00:53,700 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(132): Open test,ddd,1732071652347.63b717291dd2cfc1f752c75b18ccc26b. 2024-11-20T03:00:53,700 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/default/test/f928be6f5d7a3631384ebd87c8f80996 2024-11-20T03:00:53,700 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7752): Opening region: {ENCODED => 63b717291dd2cfc1f752c75b18ccc26b, NAME => 'test,ddd,1732071652347.63b717291dd2cfc1f752c75b18ccc26b.', STARTKEY => 'ddd', ENDKEY => 'eee'} 2024-11-20T03:00:53,701 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 63b717291dd2cfc1f752c75b18ccc26b 2024-11-20T03:00:53,701 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(898): Instantiated test,ddd,1732071652347.63b717291dd2cfc1f752c75b18ccc26b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T03:00:53,701 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7794): checking encryption for 63b717291dd2cfc1f752c75b18ccc26b 2024-11-20T03:00:53,701 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7797): checking classloading for 63b717291dd2cfc1f752c75b18ccc26b 2024-11-20T03:00:53,701 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=48, resume processing ppid=26 2024-11-20T03:00:53,701 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, ppid=26, state=SUCCESS, hasLock=false; OpenRegionProcedure 45e8fc1ea8df34f038aeb00318494097, server=1c82b83631a2,36133,1732071649570 in 317 msec 2024-11-20T03:00:53,705 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/default/test/f928be6f5d7a3631384ebd87c8f80996 2024-11-20T03:00:53,706 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1093): writing seq id for 295cabe34d2d3dbccccb06cf3aa5bbf6 2024-11-20T03:00:53,706 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1048): stopping wal replay for f928be6f5d7a3631384ebd87c8f80996 2024-11-20T03:00:53,706 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1060): Cleaning up temporary data for f928be6f5d7a3631384ebd87c8f80996 2024-11-20T03:00:53,707 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=45e8fc1ea8df34f038aeb00318494097, ASSIGN in 625 msec 2024-11-20T03:00:53,709 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1093): writing seq id for f928be6f5d7a3631384ebd87c8f80996 2024-11-20T03:00:53,710 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=49f9e405ee146272624d3d205a4ebf9e, regionState=OPEN, openSeqNum=2, regionLocation=1c82b83631a2,42975,1732071649704 2024-11-20T03:00:53,711 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39617 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=1c82b83631a2,42975,1732071649704, table=test, region=ef70776250de5f4e8ad7bd24027abe63. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-20T03:00:53,711 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/default/test/295cabe34d2d3dbccccb06cf3aa5bbf6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T03:00:53,713 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1114): Opened 295cabe34d2d3dbccccb06cf3aa5bbf6; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65211919, jitterRate=-0.028266683220863342}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T03:00:53,713 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 295cabe34d2d3dbccccb06cf3aa5bbf6 2024-11-20T03:00:53,713 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1006): Region open journal for 295cabe34d2d3dbccccb06cf3aa5bbf6: Running coprocessor pre-open hook at 1732071653679Writing region info on filesystem at 1732071653679Initializing all the Stores at 1732071653681 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732071653681Cleaning up temporary data from old regions at 1732071653697 (+16 ms)Running coprocessor post-open hooks at 1732071653713 (+16 ms)Region opened successfully at 1732071653713 2024-11-20T03:00:53,714 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=40, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure 49f9e405ee146272624d3d205a4ebf9e, server=1c82b83631a2,42975,1732071649704 because future has completed 2024-11-20T03:00:53,715 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2236): Post open deploy tasks for test,yyy,1732071652347.295cabe34d2d3dbccccb06cf3aa5bbf6., pid=49, masterSystemTime=1732071653523 2024-11-20T03:00:53,715 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=38fa6330e381ad6ba7391bf9050c289e, regionState=OPEN, openSeqNum=2, regionLocation=1c82b83631a2,43333,1732071649765 2024-11-20T03:00:53,718 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2266): Finished post open deploy task for test,yyy,1732071652347.295cabe34d2d3dbccccb06cf3aa5bbf6. 2024-11-20T03:00:53,718 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(153): Opened test,yyy,1732071652347.295cabe34d2d3dbccccb06cf3aa5bbf6. 2024-11-20T03:00:53,718 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure 38fa6330e381ad6ba7391bf9050c289e, server=1c82b83631a2,43333,1732071649765 because future has completed 2024-11-20T03:00:53,718 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(132): Open test,kkk,1732071652347.56a64fab34378d9775e0bdf29862f7e6. 2024-11-20T03:00:53,718 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7752): Opening region: {ENCODED => 56a64fab34378d9775e0bdf29862f7e6, NAME => 'test,kkk,1732071652347.56a64fab34378d9775e0bdf29862f7e6.', STARTKEY => 'kkk', ENDKEY => 'lll'} 2024-11-20T03:00:53,719 INFO [StoreOpener-63b717291dd2cfc1f752c75b18ccc26b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 63b717291dd2cfc1f752c75b18ccc26b 2024-11-20T03:00:53,719 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 56a64fab34378d9775e0bdf29862f7e6 2024-11-20T03:00:53,719 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(898): Instantiated test,kkk,1732071652347.56a64fab34378d9775e0bdf29862f7e6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T03:00:53,719 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7794): checking encryption for 56a64fab34378d9775e0bdf29862f7e6 2024-11-20T03:00:53,719 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7797): checking classloading for 56a64fab34378d9775e0bdf29862f7e6 2024-11-20T03:00:53,720 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/default/test/f928be6f5d7a3631384ebd87c8f80996/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T03:00:53,721 INFO [StoreOpener-63b717291dd2cfc1f752c75b18ccc26b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 63b717291dd2cfc1f752c75b18ccc26b columnFamilyName family 2024-11-20T03:00:53,721 INFO [StoreOpener-56a64fab34378d9775e0bdf29862f7e6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 56a64fab34378d9775e0bdf29862f7e6 2024-11-20T03:00:53,721 DEBUG [StoreOpener-63b717291dd2cfc1f752c75b18ccc26b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T03:00:53,721 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1114): Opened f928be6f5d7a3631384ebd87c8f80996; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66512915, jitterRate=-0.008880332112312317}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T03:00:53,721 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f928be6f5d7a3631384ebd87c8f80996 2024-11-20T03:00:53,722 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1006): Region open journal for f928be6f5d7a3631384ebd87c8f80996: Running coprocessor pre-open hook at 1732071653685Writing region info on filesystem at 1732071653685Initializing all the Stores at 1732071653687 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732071653687Cleaning up temporary data from old regions at 1732071653706 (+19 ms)Running coprocessor post-open hooks at 1732071653722 (+16 ms)Region opened successfully at 1732071653722 2024-11-20T03:00:53,723 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2236): Post open deploy tasks for test,iii,1732071652347.f928be6f5d7a3631384ebd87c8f80996., pid=36, masterSystemTime=1732071653406 2024-11-20T03:00:53,723 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=25 2024-11-20T03:00:53,724 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=25, state=SUCCESS, hasLock=false; OpenRegionProcedure 49f9e405ee146272624d3d205a4ebf9e, server=1c82b83631a2,42975,1732071649704 in 413 msec 2024-11-20T03:00:53,724 INFO [StoreOpener-56a64fab34378d9775e0bdf29862f7e6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 56a64fab34378d9775e0bdf29862f7e6 columnFamilyName family 2024-11-20T03:00:53,725 DEBUG [StoreOpener-56a64fab34378d9775e0bdf29862f7e6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T03:00:53,726 INFO [StoreOpener-63b717291dd2cfc1f752c75b18ccc26b-1 {}] regionserver.HStore(327): Store=63b717291dd2cfc1f752c75b18ccc26b/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T03:00:53,726 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1038): replaying wal for 63b717291dd2cfc1f752c75b18ccc26b 2024-11-20T03:00:53,726 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2266): Finished post open deploy task for test,iii,1732071652347.f928be6f5d7a3631384ebd87c8f80996. 2024-11-20T03:00:53,727 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(153): Opened test,iii,1732071652347.f928be6f5d7a3631384ebd87c8f80996. 2024-11-20T03:00:53,727 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=49f9e405ee146272624d3d205a4ebf9e, ASSIGN in 648 msec 2024-11-20T03:00:53,729 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=f928be6f5d7a3631384ebd87c8f80996, regionState=OPEN, openSeqNum=2, regionLocation=1c82b83631a2,43333,1732071649765 2024-11-20T03:00:53,733 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure f928be6f5d7a3631384ebd87c8f80996, server=1c82b83631a2,43333,1732071649765 because future has completed 2024-11-20T03:00:53,733 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/default/test/63b717291dd2cfc1f752c75b18ccc26b 2024-11-20T03:00:53,733 INFO [StoreOpener-56a64fab34378d9775e0bdf29862f7e6-1 {}] regionserver.HStore(327): Store=56a64fab34378d9775e0bdf29862f7e6/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T03:00:53,734 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1038): replaying wal for 56a64fab34378d9775e0bdf29862f7e6 2024-11-20T03:00:53,734 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/default/test/63b717291dd2cfc1f752c75b18ccc26b 2024-11-20T03:00:53,735 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1048): stopping wal replay for 63b717291dd2cfc1f752c75b18ccc26b 2024-11-20T03:00:53,735 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1060): Cleaning up temporary data for 63b717291dd2cfc1f752c75b18ccc26b 2024-11-20T03:00:53,735 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/default/test/56a64fab34378d9775e0bdf29862f7e6 2024-11-20T03:00:53,735 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39617 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=1c82b83631a2,43333,1732071649765, table=test, region=f928be6f5d7a3631384ebd87c8f80996. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-20T03:00:53,735 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/default/test/56a64fab34378d9775e0bdf29862f7e6 2024-11-20T03:00:53,736 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1048): stopping wal replay for 56a64fab34378d9775e0bdf29862f7e6 2024-11-20T03:00:53,736 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1060): Cleaning up temporary data for 56a64fab34378d9775e0bdf29862f7e6 2024-11-20T03:00:53,737 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1093): writing seq id for 63b717291dd2cfc1f752c75b18ccc26b 2024-11-20T03:00:53,739 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1093): writing seq id for 56a64fab34378d9775e0bdf29862f7e6 2024-11-20T03:00:53,742 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=17 2024-11-20T03:00:53,742 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=17, state=SUCCESS, hasLock=false; OpenRegionProcedure 38fa6330e381ad6ba7391bf9050c289e, server=1c82b83631a2,43333,1732071649765 in 473 msec 2024-11-20T03:00:53,744 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=38fa6330e381ad6ba7391bf9050c289e, ASSIGN in 666 msec 2024-11-20T03:00:53,747 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/default/test/56a64fab34378d9775e0bdf29862f7e6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T03:00:53,747 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/default/test/63b717291dd2cfc1f752c75b18ccc26b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T03:00:53,748 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1114): Opened 63b717291dd2cfc1f752c75b18ccc26b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72409030, jitterRate=0.07897862792015076}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T03:00:53,749 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 63b717291dd2cfc1f752c75b18ccc26b 2024-11-20T03:00:53,749 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=13 2024-11-20T03:00:53,749 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1006): Region open journal for 63b717291dd2cfc1f752c75b18ccc26b: Running coprocessor pre-open hook at 1732071653709Writing region info on filesystem at 1732071653709Initializing all the Stores at 1732071653711 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732071653711Cleaning up temporary data from old regions at 1732071653735 (+24 ms)Running coprocessor post-open hooks at 1732071653749 (+14 ms)Region opened successfully at 1732071653749 2024-11-20T03:00:53,749 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=13, state=SUCCESS, hasLock=false; OpenRegionProcedure f928be6f5d7a3631384ebd87c8f80996, server=1c82b83631a2,43333,1732071649765 in 463 msec 2024-11-20T03:00:53,749 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1114): Opened 56a64fab34378d9775e0bdf29862f7e6; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66067709, jitterRate=-0.015514418482780457}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T03:00:53,749 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 56a64fab34378d9775e0bdf29862f7e6 2024-11-20T03:00:53,749 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1006): Region open journal for 56a64fab34378d9775e0bdf29862f7e6: Running coprocessor pre-open hook at 1732071653719Writing region info on filesystem at 1732071653719Initializing all the Stores at 1732071653720 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732071653721 (+1 ms)Cleaning up temporary data from old regions at 1732071653736 (+15 ms)Running coprocessor post-open hooks at 1732071653749 (+13 ms)Region opened successfully at 1732071653749 2024-11-20T03:00:53,750 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ddd,1732071652347.63b717291dd2cfc1f752c75b18ccc26b., pid=46, masterSystemTime=1732071653423 2024-11-20T03:00:53,750 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2236): Post open deploy tasks for test,kkk,1732071652347.56a64fab34378d9775e0bdf29862f7e6., pid=54, masterSystemTime=1732071653523 2024-11-20T03:00:53,752 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=f928be6f5d7a3631384ebd87c8f80996, ASSIGN in 673 msec 2024-11-20T03:00:53,754 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ddd,1732071652347.63b717291dd2cfc1f752c75b18ccc26b. 2024-11-20T03:00:53,754 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(153): Opened test,ddd,1732071652347.63b717291dd2cfc1f752c75b18ccc26b. 2024-11-20T03:00:53,754 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(132): Open test,,1732071652347.f63a07d19bc4399768e352bd9fcf3523. 2024-11-20T03:00:53,754 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=63b717291dd2cfc1f752c75b18ccc26b, regionState=OPEN, openSeqNum=2, regionLocation=1c82b83631a2,42975,1732071649704 2024-11-20T03:00:53,754 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7752): Opening region: {ENCODED => f63a07d19bc4399768e352bd9fcf3523, NAME => 'test,,1732071652347.f63a07d19bc4399768e352bd9fcf3523.', STARTKEY => '', ENDKEY => 'bbb'} 2024-11-20T03:00:53,755 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test f63a07d19bc4399768e352bd9fcf3523 2024-11-20T03:00:53,755 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2266): Finished post open deploy task for test,kkk,1732071652347.56a64fab34378d9775e0bdf29862f7e6. 2024-11-20T03:00:53,755 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(898): Instantiated test,,1732071652347.f63a07d19bc4399768e352bd9fcf3523.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T03:00:53,755 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(153): Opened test,kkk,1732071652347.56a64fab34378d9775e0bdf29862f7e6. 2024-11-20T03:00:53,755 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7794): checking encryption for f63a07d19bc4399768e352bd9fcf3523 2024-11-20T03:00:53,755 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7797): checking classloading for f63a07d19bc4399768e352bd9fcf3523 2024-11-20T03:00:53,755 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(132): Open test,lll,1732071652347.535a88ef9d35d9795c43261c431087a8. 2024-11-20T03:00:53,755 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7752): Opening region: {ENCODED => 535a88ef9d35d9795c43261c431087a8, NAME => 'test,lll,1732071652347.535a88ef9d35d9795c43261c431087a8.', STARTKEY => 'lll', ENDKEY => 'mmm'} 2024-11-20T03:00:53,756 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 535a88ef9d35d9795c43261c431087a8 2024-11-20T03:00:53,756 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(898): Instantiated test,lll,1732071652347.535a88ef9d35d9795c43261c431087a8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T03:00:53,756 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7794): checking encryption for 535a88ef9d35d9795c43261c431087a8 2024-11-20T03:00:53,756 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7797): checking classloading for 535a88ef9d35d9795c43261c431087a8 2024-11-20T03:00:53,758 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=46, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 63b717291dd2cfc1f752c75b18ccc26b, server=1c82b83631a2,42975,1732071649704 because future has completed 2024-11-20T03:00:53,763 INFO [StoreOpener-f63a07d19bc4399768e352bd9fcf3523-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region f63a07d19bc4399768e352bd9fcf3523 2024-11-20T03:00:53,763 INFO [StoreOpener-535a88ef9d35d9795c43261c431087a8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 535a88ef9d35d9795c43261c431087a8 2024-11-20T03:00:53,763 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=46, resume processing ppid=8 2024-11-20T03:00:53,763 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 63b717291dd2cfc1f752c75b18ccc26b, server=1c82b83631a2,42975,1732071649704 in 402 msec 2024-11-20T03:00:53,765 INFO [StoreOpener-f63a07d19bc4399768e352bd9fcf3523-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f63a07d19bc4399768e352bd9fcf3523 columnFamilyName family 2024-11-20T03:00:53,765 INFO [StoreOpener-535a88ef9d35d9795c43261c431087a8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 535a88ef9d35d9795c43261c431087a8 columnFamilyName family 2024-11-20T03:00:53,765 DEBUG [StoreOpener-535a88ef9d35d9795c43261c431087a8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T03:00:53,765 DEBUG [StoreOpener-f63a07d19bc4399768e352bd9fcf3523-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T03:00:53,766 INFO [StoreOpener-f63a07d19bc4399768e352bd9fcf3523-1 {}] regionserver.HStore(327): Store=f63a07d19bc4399768e352bd9fcf3523/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T03:00:53,766 INFO [StoreOpener-535a88ef9d35d9795c43261c431087a8-1 {}] regionserver.HStore(327): Store=535a88ef9d35d9795c43261c431087a8/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T03:00:53,766 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=63b717291dd2cfc1f752c75b18ccc26b, ASSIGN in 687 msec 2024-11-20T03:00:53,766 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1038): replaying wal for 535a88ef9d35d9795c43261c431087a8 2024-11-20T03:00:53,767 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1038): replaying wal for f63a07d19bc4399768e352bd9fcf3523 2024-11-20T03:00:53,767 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/default/test/535a88ef9d35d9795c43261c431087a8 2024-11-20T03:00:53,768 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/default/test/f63a07d19bc4399768e352bd9fcf3523 2024-11-20T03:00:53,768 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/default/test/535a88ef9d35d9795c43261c431087a8 2024-11-20T03:00:53,768 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/default/test/f63a07d19bc4399768e352bd9fcf3523 2024-11-20T03:00:53,769 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1048): stopping wal replay for 535a88ef9d35d9795c43261c431087a8 2024-11-20T03:00:53,769 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1060): Cleaning up temporary data for 535a88ef9d35d9795c43261c431087a8 2024-11-20T03:00:53,769 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1048): stopping wal replay for f63a07d19bc4399768e352bd9fcf3523 2024-11-20T03:00:53,769 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1060): Cleaning up temporary data for f63a07d19bc4399768e352bd9fcf3523 2024-11-20T03:00:53,769 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=56a64fab34378d9775e0bdf29862f7e6, regionState=OPEN, openSeqNum=2, regionLocation=1c82b83631a2,36133,1732071649570 2024-11-20T03:00:53,771 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1093): writing seq id for f63a07d19bc4399768e352bd9fcf3523 2024-11-20T03:00:53,771 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1093): writing seq id for 535a88ef9d35d9795c43261c431087a8 2024-11-20T03:00:53,774 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=54, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure 56a64fab34378d9775e0bdf29862f7e6, server=1c82b83631a2,36133,1732071649570 because future has completed 2024-11-20T03:00:53,775 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=295cabe34d2d3dbccccb06cf3aa5bbf6, regionState=OPEN, openSeqNum=2, regionLocation=1c82b83631a2,36133,1732071649570 2024-11-20T03:00:53,775 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/default/test/535a88ef9d35d9795c43261c431087a8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T03:00:53,776 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1114): Opened 535a88ef9d35d9795c43261c431087a8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73336091, jitterRate=0.09279291331768036}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T03:00:53,776 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 535a88ef9d35d9795c43261c431087a8 2024-11-20T03:00:53,776 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1006): Region open journal for 535a88ef9d35d9795c43261c431087a8: Running coprocessor pre-open hook at 1732071653756Writing region info on filesystem at 1732071653756Initializing all the Stores at 1732071653758 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732071653758Cleaning up temporary data from old regions at 1732071653769 (+11 ms)Running coprocessor post-open hooks at 1732071653776 (+7 ms)Region opened successfully at 1732071653776 2024-11-20T03:00:53,778 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2236): Post open deploy tasks for test,lll,1732071652347.535a88ef9d35d9795c43261c431087a8., pid=55, masterSystemTime=1732071653523 2024-11-20T03:00:53,779 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=49, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 295cabe34d2d3dbccccb06cf3aa5bbf6, server=1c82b83631a2,36133,1732071649570 because future has completed 2024-11-20T03:00:53,783 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/default/test/f63a07d19bc4399768e352bd9fcf3523/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T03:00:53,784 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2266): Finished post open deploy task for test,lll,1732071652347.535a88ef9d35d9795c43261c431087a8. 2024-11-20T03:00:53,784 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(153): Opened test,lll,1732071652347.535a88ef9d35d9795c43261c431087a8. 2024-11-20T03:00:53,784 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1114): Opened f63a07d19bc4399768e352bd9fcf3523; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74517085, jitterRate=0.11039109528064728}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T03:00:53,784 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f63a07d19bc4399768e352bd9fcf3523 2024-11-20T03:00:53,784 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(132): Open test,bbb,1732071652347.edd7cafc43290cb2489e8ba273907acd. 2024-11-20T03:00:53,784 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1006): Region open journal for f63a07d19bc4399768e352bd9fcf3523: Running coprocessor pre-open hook at 1732071653755Writing region info on filesystem at 1732071653755Initializing all the Stores at 1732071653757 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732071653757Cleaning up temporary data from old regions at 1732071653769 (+12 ms)Running coprocessor post-open hooks at 1732071653784 (+15 ms)Region opened successfully at 1732071653784 2024-11-20T03:00:53,784 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7752): Opening region: {ENCODED => edd7cafc43290cb2489e8ba273907acd, NAME => 'test,bbb,1732071652347.edd7cafc43290cb2489e8ba273907acd.', STARTKEY => 'bbb', ENDKEY => 'ccc'} 2024-11-20T03:00:53,784 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test edd7cafc43290cb2489e8ba273907acd 2024-11-20T03:00:53,784 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(898): Instantiated test,bbb,1732071652347.edd7cafc43290cb2489e8ba273907acd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T03:00:53,785 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7794): checking encryption for edd7cafc43290cb2489e8ba273907acd 2024-11-20T03:00:53,785 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7797): checking classloading for edd7cafc43290cb2489e8ba273907acd 2024-11-20T03:00:53,785 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2236): Post open deploy tasks for test,,1732071652347.f63a07d19bc4399768e352bd9fcf3523., pid=50, masterSystemTime=1732071653423 2024-11-20T03:00:53,787 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=54, resume processing ppid=15 2024-11-20T03:00:53,787 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, ppid=15, state=SUCCESS, hasLock=false; OpenRegionProcedure 56a64fab34378d9775e0bdf29862f7e6, server=1c82b83631a2,36133,1732071649570 in 376 msec 2024-11-20T03:00:53,787 INFO [StoreOpener-edd7cafc43290cb2489e8ba273907acd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region edd7cafc43290cb2489e8ba273907acd 2024-11-20T03:00:53,788 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2266): Finished post open deploy task for test,,1732071652347.f63a07d19bc4399768e352bd9fcf3523. 2024-11-20T03:00:53,788 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(153): Opened test,,1732071652347.f63a07d19bc4399768e352bd9fcf3523. 2024-11-20T03:00:53,789 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=56a64fab34378d9775e0bdf29862f7e6, ASSIGN in 711 msec 2024-11-20T03:00:53,790 INFO [StoreOpener-edd7cafc43290cb2489e8ba273907acd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region edd7cafc43290cb2489e8ba273907acd columnFamilyName family 2024-11-20T03:00:53,790 DEBUG [StoreOpener-edd7cafc43290cb2489e8ba273907acd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T03:00:53,791 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=f63a07d19bc4399768e352bd9fcf3523, regionState=OPEN, openSeqNum=2, regionLocation=1c82b83631a2,42975,1732071649704 2024-11-20T03:00:53,791 INFO [StoreOpener-edd7cafc43290cb2489e8ba273907acd-1 {}] regionserver.HStore(327): Store=edd7cafc43290cb2489e8ba273907acd/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T03:00:53,791 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1038): replaying wal for edd7cafc43290cb2489e8ba273907acd 2024-11-20T03:00:53,793 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/default/test/edd7cafc43290cb2489e8ba273907acd 2024-11-20T03:00:53,793 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/default/test/edd7cafc43290cb2489e8ba273907acd 2024-11-20T03:00:53,794 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1048): stopping wal replay for edd7cafc43290cb2489e8ba273907acd 2024-11-20T03:00:53,794 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1060): Cleaning up temporary data for edd7cafc43290cb2489e8ba273907acd 2024-11-20T03:00:53,795 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=50, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure f63a07d19bc4399768e352bd9fcf3523, server=1c82b83631a2,42975,1732071649704 because future has completed 2024-11-20T03:00:53,796 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=535a88ef9d35d9795c43261c431087a8, regionState=OPEN, openSeqNum=2, regionLocation=1c82b83631a2,36133,1732071649570 2024-11-20T03:00:53,796 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1093): writing seq id for edd7cafc43290cb2489e8ba273907acd 2024-11-20T03:00:53,799 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=55, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 535a88ef9d35d9795c43261c431087a8, server=1c82b83631a2,36133,1732071649570 because future has completed 2024-11-20T03:00:53,799 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/default/test/edd7cafc43290cb2489e8ba273907acd/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T03:00:53,800 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1114): Opened edd7cafc43290cb2489e8ba273907acd; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69837638, jitterRate=0.040661901235580444}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T03:00:53,800 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1122): Running coprocessor post-open hooks for edd7cafc43290cb2489e8ba273907acd 2024-11-20T03:00:53,801 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1006): Region open journal for edd7cafc43290cb2489e8ba273907acd: Running coprocessor pre-open hook at 1732071653785Writing region info on filesystem at 1732071653785Initializing all the Stores at 1732071653786 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732071653787 (+1 ms)Cleaning up temporary data from old regions at 1732071653794 (+7 ms)Running coprocessor post-open hooks at 1732071653800 (+6 ms)Region opened successfully at 1732071653800 2024-11-20T03:00:53,802 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2236): Post open deploy tasks for test,bbb,1732071652347.edd7cafc43290cb2489e8ba273907acd., pid=56, masterSystemTime=1732071653523 2024-11-20T03:00:53,804 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2266): Finished post open deploy task for test,bbb,1732071652347.edd7cafc43290cb2489e8ba273907acd. 2024-11-20T03:00:53,804 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(153): Opened test,bbb,1732071652347.edd7cafc43290cb2489e8ba273907acd. 2024-11-20T03:00:53,804 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(132): Open test,rrr,1732071652347.a641a5878ac82d20c96fa3dff0720791. 2024-11-20T03:00:53,804 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7752): Opening region: {ENCODED => a641a5878ac82d20c96fa3dff0720791, NAME => 'test,rrr,1732071652347.a641a5878ac82d20c96fa3dff0720791.', STARTKEY => 'rrr', ENDKEY => 'sss'} 2024-11-20T03:00:53,805 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test a641a5878ac82d20c96fa3dff0720791 2024-11-20T03:00:53,805 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(898): Instantiated test,rrr,1732071652347.a641a5878ac82d20c96fa3dff0720791.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T03:00:53,805 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7794): checking encryption for a641a5878ac82d20c96fa3dff0720791 2024-11-20T03:00:53,805 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7797): checking classloading for a641a5878ac82d20c96fa3dff0720791 2024-11-20T03:00:53,806 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=50, resume processing ppid=5 2024-11-20T03:00:53,806 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure f63a07d19bc4399768e352bd9fcf3523, server=1c82b83631a2,42975,1732071649704 in 418 msec 2024-11-20T03:00:53,807 INFO [StoreOpener-a641a5878ac82d20c96fa3dff0720791-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region a641a5878ac82d20c96fa3dff0720791 2024-11-20T03:00:53,809 INFO [StoreOpener-a641a5878ac82d20c96fa3dff0720791-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a641a5878ac82d20c96fa3dff0720791 columnFamilyName family 2024-11-20T03:00:53,809 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=f63a07d19bc4399768e352bd9fcf3523, ASSIGN in 730 msec 2024-11-20T03:00:53,809 DEBUG [StoreOpener-a641a5878ac82d20c96fa3dff0720791-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T03:00:53,810 INFO [StoreOpener-a641a5878ac82d20c96fa3dff0720791-1 {}] regionserver.HStore(327): Store=a641a5878ac82d20c96fa3dff0720791/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T03:00:53,810 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1038): replaying wal for a641a5878ac82d20c96fa3dff0720791 2024-11-20T03:00:53,811 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/default/test/a641a5878ac82d20c96fa3dff0720791 2024-11-20T03:00:53,813 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/default/test/a641a5878ac82d20c96fa3dff0720791 2024-11-20T03:00:53,813 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=edd7cafc43290cb2489e8ba273907acd, regionState=OPEN, openSeqNum=2, regionLocation=1c82b83631a2,36133,1732071649570 2024-11-20T03:00:53,813 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39617 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=1c82b83631a2,36133,1732071649570, table=test, region=edd7cafc43290cb2489e8ba273907acd. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-20T03:00:53,813 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39617 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=1c82b83631a2,36133,1732071649570, table=test, region=2fe87cdaf67461ff5c508d851e418c66. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-20T03:00:53,813 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39617 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=1c82b83631a2,36133,1732071649570, table=test, region=535a88ef9d35d9795c43261c431087a8. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-20T03:00:53,813 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39617 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=1c82b83631a2,36133,1732071649570, table=test, region=550bbfe722915166ce74fec36828c508. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-20T03:00:53,814 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39617 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=1c82b83631a2,36133,1732071649570, table=test, region=295cabe34d2d3dbccccb06cf3aa5bbf6. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-20T03:00:53,814 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1048): stopping wal replay for a641a5878ac82d20c96fa3dff0720791 2024-11-20T03:00:53,814 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1060): Cleaning up temporary data for a641a5878ac82d20c96fa3dff0720791 2024-11-20T03:00:53,817 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1093): writing seq id for a641a5878ac82d20c96fa3dff0720791 2024-11-20T03:00:53,817 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure edd7cafc43290cb2489e8ba273907acd, server=1c82b83631a2,36133,1732071649570 because future has completed 2024-11-20T03:00:53,819 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39617 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=1c82b83631a2,42975,1732071649704, table=test, region=ef70776250de5f4e8ad7bd24027abe63. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-20T03:00:53,822 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/default/test/a641a5878ac82d20c96fa3dff0720791/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T03:00:53,823 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1114): Opened a641a5878ac82d20c96fa3dff0720791; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61471662, jitterRate=-0.0840008556842804}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T03:00:53,823 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a641a5878ac82d20c96fa3dff0720791 2024-11-20T03:00:53,823 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=16 2024-11-20T03:00:53,823 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1006): Region open journal for a641a5878ac82d20c96fa3dff0720791: Running coprocessor pre-open hook at 1732071653805Writing region info on filesystem at 1732071653805Initializing all the Stores at 1732071653806 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732071653806Cleaning up temporary data from old regions at 1732071653814 (+8 ms)Running coprocessor post-open hooks at 1732071653823 (+9 ms)Region opened successfully at 1732071653823 2024-11-20T03:00:53,823 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure 535a88ef9d35d9795c43261c431087a8, server=1c82b83631a2,36133,1732071649570 in 410 msec 2024-11-20T03:00:53,824 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2236): Post open deploy tasks for test,rrr,1732071652347.a641a5878ac82d20c96fa3dff0720791., pid=51, masterSystemTime=1732071653523 2024-11-20T03:00:53,826 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=535a88ef9d35d9795c43261c431087a8, ASSIGN in 747 msec 2024-11-20T03:00:53,827 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2266): Finished post open deploy task for test,rrr,1732071652347.a641a5878ac82d20c96fa3dff0720791. 2024-11-20T03:00:53,827 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(153): Opened test,rrr,1732071652347.a641a5878ac82d20c96fa3dff0720791. 2024-11-20T03:00:53,827 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(132): Open test,hhh,1732071652347.2816319e15be2a1534d61304db509629. 2024-11-20T03:00:53,828 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7752): Opening region: {ENCODED => 2816319e15be2a1534d61304db509629, NAME => 'test,hhh,1732071652347.2816319e15be2a1534d61304db509629.', STARTKEY => 'hhh', ENDKEY => 'iii'} 2024-11-20T03:00:53,828 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 2816319e15be2a1534d61304db509629 2024-11-20T03:00:53,828 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(898): Instantiated test,hhh,1732071652347.2816319e15be2a1534d61304db509629.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T03:00:53,829 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7794): checking encryption for 2816319e15be2a1534d61304db509629 2024-11-20T03:00:53,829 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7797): checking classloading for 2816319e15be2a1534d61304db509629 2024-11-20T03:00:53,831 INFO [StoreOpener-2816319e15be2a1534d61304db509629-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 2816319e15be2a1534d61304db509629 2024-11-20T03:00:53,839 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=6 2024-11-20T03:00:53,839 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=6, state=SUCCESS, hasLock=false; OpenRegionProcedure edd7cafc43290cb2489e8ba273907acd, server=1c82b83631a2,36133,1732071649570 in 407 msec 2024-11-20T03:00:53,845 INFO [StoreOpener-2816319e15be2a1534d61304db509629-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2816319e15be2a1534d61304db509629 columnFamilyName family 2024-11-20T03:00:53,845 DEBUG [StoreOpener-2816319e15be2a1534d61304db509629-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T03:00:53,846 INFO [StoreOpener-2816319e15be2a1534d61304db509629-1 {}] regionserver.HStore(327): Store=2816319e15be2a1534d61304db509629/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T03:00:53,847 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1038): replaying wal for 2816319e15be2a1534d61304db509629 2024-11-20T03:00:53,849 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/default/test/2816319e15be2a1534d61304db509629 2024-11-20T03:00:53,849 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=edd7cafc43290cb2489e8ba273907acd, ASSIGN in 763 msec 2024-11-20T03:00:53,850 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/default/test/2816319e15be2a1534d61304db509629 2024-11-20T03:00:53,851 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1048): stopping wal replay for 2816319e15be2a1534d61304db509629 2024-11-20T03:00:53,851 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1060): Cleaning up temporary data for 2816319e15be2a1534d61304db509629 2024-11-20T03:00:53,852 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=a641a5878ac82d20c96fa3dff0720791, regionState=OPEN, openSeqNum=2, regionLocation=1c82b83631a2,36133,1732071649570 2024-11-20T03:00:53,856 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=51, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure a641a5878ac82d20c96fa3dff0720791, server=1c82b83631a2,36133,1732071649570 because future has completed 2024-11-20T03:00:53,858 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1093): writing seq id for 2816319e15be2a1534d61304db509629 2024-11-20T03:00:53,863 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/default/test/2816319e15be2a1534d61304db509629/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T03:00:53,864 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1114): Opened 2816319e15be2a1534d61304db509629; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72754823, jitterRate=0.08413134515285492}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T03:00:53,864 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2816319e15be2a1534d61304db509629 2024-11-20T03:00:53,864 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1006): Region open journal for 2816319e15be2a1534d61304db509629: Running coprocessor pre-open hook at 1732071653829Writing region info on filesystem at 1732071653829Initializing all the Stores at 1732071653830 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732071653831 (+1 ms)Cleaning up temporary data from old regions at 1732071653852 (+21 ms)Running coprocessor post-open hooks at 1732071653864 (+12 ms)Region opened successfully at 1732071653864 2024-11-20T03:00:53,865 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2236): Post open deploy tasks for test,hhh,1732071652347.2816319e15be2a1534d61304db509629., pid=53, masterSystemTime=1732071653523 2024-11-20T03:00:53,867 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=29 2024-11-20T03:00:53,867 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure 295cabe34d2d3dbccccb06cf3aa5bbf6, server=1c82b83631a2,36133,1732071649570 in 480 msec 2024-11-20T03:00:53,869 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2266): Finished post open deploy task for test,hhh,1732071652347.2816319e15be2a1534d61304db509629. 2024-11-20T03:00:53,869 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(153): Opened test,hhh,1732071652347.2816319e15be2a1534d61304db509629. 2024-11-20T03:00:53,870 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=295cabe34d2d3dbccccb06cf3aa5bbf6, ASSIGN in 791 msec 2024-11-20T03:00:53,872 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=2816319e15be2a1534d61304db509629, regionState=OPEN, openSeqNum=2, regionLocation=1c82b83631a2,36133,1732071649570 2024-11-20T03:00:53,876 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=53, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2816319e15be2a1534d61304db509629, server=1c82b83631a2,36133,1732071649570 because future has completed 2024-11-20T03:00:53,882 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=22 2024-11-20T03:00:53,882 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=22, state=SUCCESS, hasLock=false; OpenRegionProcedure a641a5878ac82d20c96fa3dff0720791, server=1c82b83631a2,36133,1732071649570 in 487 msec 2024-11-20T03:00:53,886 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=a641a5878ac82d20c96fa3dff0720791, ASSIGN in 806 msec 2024-11-20T03:00:53,892 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=12 2024-11-20T03:00:53,892 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=12, state=SUCCESS, hasLock=false; OpenRegionProcedure 2816319e15be2a1534d61304db509629, server=1c82b83631a2,36133,1732071649570 in 488 msec 2024-11-20T03:00:53,894 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=2816319e15be2a1534d61304db509629, ASSIGN in 816 msec 2024-11-20T03:00:53,901 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=52, resume processing ppid=9 2024-11-20T03:00:53,901 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, ppid=9, state=SUCCESS, hasLock=false; OpenRegionProcedure 2fe87cdaf67461ff5c508d851e418c66, server=1c82b83631a2,36133,1732071649570 in 501 msec 2024-11-20T03:00:53,905 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=2fe87cdaf67461ff5c508d851e418c66, ASSIGN in 826 msec 2024-11-20T03:00:53,910 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=44, resume processing ppid=20 2024-11-20T03:00:53,910 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure ef70776250de5f4e8ad7bd24027abe63, server=1c82b83631a2,42975,1732071649704 in 568 msec 2024-11-20T03:00:53,913 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=ef70776250de5f4e8ad7bd24027abe63, ASSIGN in 834 msec 2024-11-20T03:00:53,923 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=23 2024-11-20T03:00:53,923 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=23, state=SUCCESS, hasLock=false; OpenRegionProcedure 550bbfe722915166ce74fec36828c508, server=1c82b83631a2,36133,1732071649570 in 545 msec 2024-11-20T03:00:53,928 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=23, resume processing ppid=4 2024-11-20T03:00:53,928 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=550bbfe722915166ce74fec36828c508, ASSIGN in 847 msec 2024-11-20T03:00:53,930 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-20T03:00:53,931 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732071653930"}]},"ts":"1732071653930"} 2024-11-20T03:00:53,934 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=test, state=ENABLED in hbase:meta 2024-11-20T03:00:53,936 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_POST_OPERATION 2024-11-20T03:00:53,945 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test in 1.5800 sec 2024-11-20T03:00:54,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39617 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T03:00:54,522 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table test get assigned. Timeout = 60000ms 2024-11-20T03:00:54,522 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:test completed 2024-11-20T03:00:54,525 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T03:00:54,546 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table test assigned to meta. Checking AM states. 2024-11-20T03:00:54,546 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T03:00:54,547 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table test assigned. 2024-11-20T03:00:54,547 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T03:00:54,552 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test,, stopping at row=test ,, for max=2147483647 with caching=100 2024-11-20T03:00:54,576 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test,, stopping at row=test ,, for max=2147483647 with caching=100 2024-11-20T03:00:54,588 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T03:00:54,591 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58784, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T03:00:54,593 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T03:00:54,595 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38548, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T03:00:54,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39617 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region edd7cafc43290cb2489e8ba273907acd changed from -1.0 to 0.0, refreshing cache 2024-11-20T03:00:54,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39617 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 2fe87cdaf67461ff5c508d851e418c66 changed from -1.0 to 0.0, refreshing cache 2024-11-20T03:00:54,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39617 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 2816319e15be2a1534d61304db509629 changed from -1.0 to 0.0, refreshing cache 2024-11-20T03:00:54,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39617 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 56a64fab34378d9775e0bdf29862f7e6 changed from -1.0 to 0.0, refreshing cache 2024-11-20T03:00:54,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39617 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 535a88ef9d35d9795c43261c431087a8 changed from -1.0 to 0.0, refreshing cache 2024-11-20T03:00:54,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39617 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region a641a5878ac82d20c96fa3dff0720791 changed from -1.0 to 0.0, refreshing cache 2024-11-20T03:00:54,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39617 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 550bbfe722915166ce74fec36828c508 changed from -1.0 to 0.0, refreshing cache 2024-11-20T03:00:54,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39617 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 45e8fc1ea8df34f038aeb00318494097 changed from -1.0 to 0.0, refreshing cache 2024-11-20T03:00:54,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39617 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 295cabe34d2d3dbccccb06cf3aa5bbf6 changed from -1.0 to 0.0, refreshing cache 2024-11-20T03:00:54,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39617 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region f63a07d19bc4399768e352bd9fcf3523 changed from -1.0 to 0.0, refreshing cache 2024-11-20T03:00:54,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39617 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 63b717291dd2cfc1f752c75b18ccc26b changed from -1.0 to 0.0, refreshing cache 2024-11-20T03:00:54,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39617 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 3c5e70743a336dce38e44ee0f8059378 changed from -1.0 to 0.0, refreshing cache 2024-11-20T03:00:54,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39617 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region ad7c736680f4a75df049e62d21f77c48 changed from -1.0 to 0.0, refreshing cache 2024-11-20T03:00:54,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39617 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region fbf7ef74de7c98db6b37604b08e67282 changed from -1.0 to 0.0, refreshing cache 2024-11-20T03:00:54,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39617 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region ef70776250de5f4e8ad7bd24027abe63 changed from -1.0 to 0.0, refreshing cache 2024-11-20T03:00:54,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39617 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 30cc99ff17ee8d21b793ce798b4998ba changed from -1.0 to 0.0, refreshing cache 2024-11-20T03:00:54,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39617 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 49f9e405ee146272624d3d205a4ebf9e changed from -1.0 to 0.0, refreshing cache 2024-11-20T03:00:54,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39617 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region e8e6d28d8e1806fa7d958327b87df211 changed from -1.0 to 0.0, refreshing cache 2024-11-20T03:00:54,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39617 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region ccd7aea6ec14848f8c6730c88ce1c780 changed from -1.0 to 0.0, refreshing cache 2024-11-20T03:00:54,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39617 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region fc28de84807b58cb0d6ceaa73127debb changed from -1.0 to 0.0, refreshing cache 2024-11-20T03:00:54,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39617 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region f928be6f5d7a3631384ebd87c8f80996 changed from -1.0 to 0.0, refreshing cache 2024-11-20T03:00:54,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39617 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 38fa6330e381ad6ba7391bf9050c289e changed from -1.0 to 0.0, refreshing cache 2024-11-20T03:00:54,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39617 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 4327d986b4dd716142908b96606b0ad7 changed from -1.0 to 0.0, refreshing cache 2024-11-20T03:00:54,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39617 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 8a818d7f8283ee7f2b5fae844b64a8ee changed from -1.0 to 0.0, refreshing cache 2024-11-20T03:00:54,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39617 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 8a06021a0c54a875234d29e87253d48a changed from -1.0 to 0.0, refreshing cache 2024-11-20T03:00:54,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39617 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region bf73c409fe50a78c430f016d1431abfb changed from -1.0 to 0.0, refreshing cache 2024-11-20T03:00:54,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39617 {}] rsgroup.RSGroupBasedLoadBalancer(138): Balancing RSGroup=default 2024-11-20T03:00:54,666 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39617 {}] rsgroup.RSGroupBasedLoadBalancer(151): Start Generate Balance plan for group: default 2024-11-20T03:00:54,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39617 {}] balancer.BaseLoadBalancer(619): Start Generate Balance plan for cluster. 2024-11-20T03:00:54,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39617 {}] balancer.BalancerClusterState(204): Hosts are {1c82b83631a2=0} racks are {/default-rack=0} 2024-11-20T03:00:54,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39617 {}] balancer.BalancerClusterState(303): server 0 has 9 regions 2024-11-20T03:00:54,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39617 {}] balancer.BalancerClusterState(303): server 1 has 9 regions 2024-11-20T03:00:54,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39617 {}] balancer.BalancerClusterState(303): server 2 has 9 regions 2024-11-20T03:00:54,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39617 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-20T03:00:54,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39617 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-20T03:00:54,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39617 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-20T03:00:54,668 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39617 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-20T03:00:54,668 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39617 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-20T03:00:54,668 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39617 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-20T03:00:54,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39617 {}] balancer.BalancerClusterState(326): Number of tables=2, number of hosts=1, number of racks=1 2024-11-20T03:00:54,675 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39617 {}] balancer.StochasticLoadBalancer(395): Cluster wide - skipping load balancing because weighted average imbalance=0.016474464579901153 <= threshold(0.025). If you want more aggressive balancing, either lower hbase.master.balancer.stochastic.minCostNeedBalance from 0.025 or increase the relative multiplier(s) of the specific cost function(s). functionCost=RegionCountSkewCostFunction : (multiplier=500.0, imbalance=0.0); PrimaryRegionCountSkewCostFunction : (not needed); MoveCostFunction : (multiplier=7.0, imbalance=0.0); ServerLocalityCostFunction : (multiplier=25.0, imbalance=0.0); RackLocalityCostFunction : (multiplier=15.0, imbalance=0.0); TableSkewCostFunction : (multiplier=35.0, imbalance=0.0); RegionReplicaHostCostFunction : (not needed); RegionReplicaRackCostFunction : (not needed); ReadRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); CPRequestCostFunction : (multiplier=5.0, imbalance=0.0); WriteRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); MemStoreSizeCostFunction : (multiplier=5.0, imbalance=0.0); StoreFileCostFunction : (multiplier=5.0, imbalance=0.0); 2024-11-20T03:00:54,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39617 {}] master.HMaster(2203): Balancer is going into sleep until next period in 300000ms 2024-11-20T03:00:54,678 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [900,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T03:00:54,689 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39617 {}] master.MasterRpcServices(567): Client=jenkins//172.17.0.2 set balanceSwitch=false 2024-11-20T03:00:54,712 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: master.assignment.TestRollbackSCP#testFailAndRollback Thread=382, OpenFileDescriptor=641, MaxFileDescriptor=1048576, SystemLoadAverage=190, ProcessCount=11, AvailableMemoryMB=6440 2024-11-20T03:00:54,715 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table hbase:meta 2024-11-20T03:00:54,715 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=hbase:meta,,1 2024-11-20T03:00:54,716 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(267): Killing 1c82b83631a2,43333,1732071649765 2024-11-20T03:00:54,716 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server 1c82b83631a2,43333,1732071649765: Simulated kill ***** 2024-11-20T03:00:54,716 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-20T03:00:54,718 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-20T03:00:54,720 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-20T03:00:54,721 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-20T03:00:54,721 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-20T03:00:54,721 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 300445408 }, "NonHeapMemoryUsage": { "committed": 152305664, "init": 7667712, "max": -1, "used": 149945816 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "1c82b83631a2" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "1c82b83631a2" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.Context": "regionserver", "tag.Hostname": "1c82b83631a2" } ] } 2024-11-20T03:00:54,724 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39617 {}] master.MasterRpcServices(700): 1c82b83631a2,43333,1732071649765 reported a fatal error: ***** ABORTING region server 1c82b83631a2,43333,1732071649765: Simulated kill ***** 2024-11-20T03:00:54,726 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '1c82b83631a2,43333,1732071649765' ***** 2024-11-20T03:00:54,726 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Simulated kill 2024-11-20T03:00:54,726 INFO [RS:2;1c82b83631a2:43333 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-20T03:00:54,726 INFO [RS:2;1c82b83631a2:43333 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-11-20T03:00:54,726 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-20T03:00:54,727 INFO [RS:2;1c82b83631a2:43333 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-11-20T03:00:54,727 INFO [RS:2;1c82b83631a2:43333 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-20T03:00:54,727 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [15,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T03:00:54,727 INFO [RS:2;1c82b83631a2:43333 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;1c82b83631a2:43333. 2024-11-20T03:00:54,727 DEBUG [RS:2;1c82b83631a2:43333 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T03:00:54,731 DEBUG [RS:2;1c82b83631a2:43333 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T03:00:54,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42923 is added to blk_1073741836_1012 (size=32114) 2024-11-20T03:00:54,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33951 is added to blk_1073741836_1012 (size=32114) 2024-11-20T03:00:54,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38873 is added to blk_1073741836_1012 (size=32114) 2024-11-20T03:00:54,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33951 is added to blk_1073741834_1010 (size=2237) 2024-11-20T03:00:54,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42923 is added to blk_1073741834_1010 (size=2237) 2024-11-20T03:00:54,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38873 is added to blk_1073741834_1010 (size=2237) 2024-11-20T03:00:54,742 DEBUG [RS:2;1c82b83631a2:43333 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T03:00:54,742 INFO [RS:2;1c82b83631a2:43333 {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T03:00:54,742 INFO [RS:2;1c82b83631a2:43333 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43333 2024-11-20T03:00:54,805 INFO [regionserver/1c82b83631a2:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T03:00:54,807 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39617-0x10157797d910000, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-20T03:00:54,807 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43333-0x10157797d910003, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/1c82b83631a2,43333,1732071649765 2024-11-20T03:00:54,807 INFO [RS:2;1c82b83631a2:43333 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-20T03:00:54,809 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [1c82b83631a2,43333,1732071649765] 2024-11-20T03:00:54,824 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/1c82b83631a2,43333,1732071649765 already deleted, retry=false 2024-11-20T03:00:54,824 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 1c82b83631a2,43333,1732071649765 on 1c82b83631a2,39617,1732071648799 2024-11-20T03:00:54,829 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=57, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 1c82b83631a2,43333,1732071649765, splitWal=true, meta=true 2024-11-20T03:00:54,833 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=57 for 1c82b83631a2,43333,1732071649765 (carryingMeta=true) 1c82b83631a2,43333,1732071649765/CRASHED/regionCount=9/lock=java.util.concurrent.locks.ReentrantReadWriteLock@7e34e9c7[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-20T03:00:54,835 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=57, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 1c82b83631a2,43333,1732071649765, splitWal=true, meta=true 2024-11-20T03:00:54,836 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 1c82b83631a2,43333,1732071649765, splitWal=true, meta=true, isMeta: true 2024-11-20T03:00:54,839 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/WALs/1c82b83631a2,43333,1732071649765-splitting 2024-11-20T03:00:54,841 INFO [PEWorker-1 {}] master.SplitWALManager(105): 1c82b83631a2,43333,1732071649765 WAL count=1, meta=true 2024-11-20T03:00:54,844 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 1c82b83631a2%2C43333%2C1732071649765.meta.1732071651840.meta}] 2024-11-20T03:00:54,850 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=1c82b83631a2,36133,1732071649570 2024-11-20T03:00:54,853 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 1c82b83631a2%2C43333%2C1732071649765.meta.1732071651840.meta, worker=1c82b83631a2,36133,1732071649570}] 2024-11-20T03:00:54,916 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43333-0x10157797d910003, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T03:00:54,916 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43333-0x10157797d910003, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T03:00:54,917 INFO [RS:2;1c82b83631a2:43333 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-20T03:00:54,917 INFO [RS:2;1c82b83631a2:43333 {}] regionserver.HRegionServer(1031): Exiting; stopping=1c82b83631a2,43333,1732071649765; zookeeper connection closed. 2024-11-20T03:00:54,917 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@45e10e2a {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@45e10e2a 2024-11-20T03:00:54,928 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T03:00:55,021 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=59 2024-11-20T03:00:55,049 INFO [RS_LOG_REPLAY_OPS-regionserver/1c82b83631a2:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(299): Splitting hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/WALs/1c82b83631a2,43333,1732071649765-splitting/1c82b83631a2%2C43333%2C1732071649765.meta.1732071651840.meta, size=31.4 K (32114bytes) 2024-11-20T03:00:55,050 DEBUG [RS_LOG_REPLAY_OPS-regionserver/1c82b83631a2:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-11-20T03:00:55,050 DEBUG [RS_LOG_REPLAY_OPS-regionserver/1c82b83631a2:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-11-20T03:00:55,050 INFO [RS_LOG_REPLAY_OPS-regionserver/1c82b83631a2:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/WALs/1c82b83631a2,43333,1732071649765-splitting/1c82b83631a2%2C43333%2C1732071649765.meta.1732071651840.meta 2024-11-20T03:00:55,055 INFO [RS_LOG_REPLAY_OPS-regionserver/1c82b83631a2:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/WALs/1c82b83631a2,43333,1732071649765-splitting/1c82b83631a2%2C43333%2C1732071649765.meta.1732071651840.meta after 4ms 2024-11-20T03:00:55,063 INFO [RS_LOG_REPLAY_OPS-regionserver/1c82b83631a2:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(310): Open hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/WALs/1c82b83631a2,43333,1732071649765-splitting/1c82b83631a2%2C43333%2C1732071649765.meta.1732071651840.meta took 15ms 2024-11-20T03:00:55,073 DEBUG [RS_LOG_REPLAY_OPS-regionserver/1c82b83631a2:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(352): Last flushed sequenceid for 1588230740: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "info" sequence_id: 5 } store_sequence_id { family_name: "ns" sequence_id: 3 } store_sequence_id { family_name: "rep_barrier" sequence_id: 18446744073709551615 } store_sequence_id { family_name: "table" sequence_id: 6 } 2024-11-20T03:00:55,078 INFO [RS_LOG_REPLAY_OPS-regionserver/1c82b83631a2:0-0-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-1c82b83631a2%2C43333%2C1732071649765.meta.1732071651840.meta.temp 2024-11-20T03:00:55,082 INFO [RS_LOG_REPLAY_OPS-regionserver/1c82b83631a2:0-0-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-1c82b83631a2%2C43333%2C1732071649765.meta.1732071651840.meta.temp 2024-11-20T03:00:55,085 DEBUG [RS_LOG_REPLAY_OPS-regionserver/1c82b83631a2:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/WALs/1c82b83631a2,43333,1732071649765-splitting/1c82b83631a2%2C43333%2C1732071649765.meta.1732071651840.meta so closing down 2024-11-20T03:00:55,085 DEBUG [RS_LOG_REPLAY_OPS-regionserver/1c82b83631a2:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-20T03:00:55,086 INFO [RS_LOG_REPLAY_OPS-regionserver/1c82b83631a2:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.OutputSink(145): 3 split writer threads finished 2024-11-20T03:00:55,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33951 is added to blk_1073741864_1040 (size=31873) 2024-11-20T03:00:55,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42923 is added to blk_1073741864_1040 (size=31873) 2024-11-20T03:00:55,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38873 is added to blk_1073741864_1040 (size=31873) 2024-11-20T03:00:55,096 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-1c82b83631a2%2C43333%2C1732071649765.meta.1732071651840.meta.temp (wrote 57 edits, skipped 0 edits in 3 ms) 2024-11-20T03:00:55,098 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-1c82b83631a2%2C43333%2C1732071649765.meta.1732071651840.meta.temp to hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-20T03:00:55,103 INFO [RS_LOG_REPLAY_OPS-regionserver/1c82b83631a2:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(425): Processed 58 edits across 1 Regions in 34 ms; skipped=1; WAL=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/WALs/1c82b83631a2,43333,1732071649765-splitting/1c82b83631a2%2C43333%2C1732071649765.meta.1732071651840.meta, size=31.4 K, length=32114, corrupted=false, cancelled=false 2024-11-20T03:00:55,104 DEBUG [RS_LOG_REPLAY_OPS-regionserver/1c82b83631a2:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(428): Completed split of hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/WALs/1c82b83631a2,43333,1732071649765-splitting/1c82b83631a2%2C43333%2C1732071649765.meta.1732071651840.meta, journal: Splitting hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/WALs/1c82b83631a2,43333,1732071649765-splitting/1c82b83631a2%2C43333%2C1732071649765.meta.1732071651840.meta, size=31.4 K (32114bytes) at 1732071655049Creating recovered edits writer path=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-1c82b83631a2%2C43333%2C1732071649765.meta.1732071651840.meta.temp at 1732071655082 (+33 ms)Finishing writing output for hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/WALs/1c82b83631a2,43333,1732071649765-splitting/1c82b83631a2%2C43333%2C1732071649765.meta.1732071651840.meta so closing down at 1732071655085 (+3 ms)3 split writer threads finished at 1732071655086 (+1 ms)Closed recovered edits writer path=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-1c82b83631a2%2C43333%2C1732071649765.meta.1732071651840.meta.temp (wrote 57 edits, skipped 0 edits in 3 ms) at 1732071655096 (+10 ms)Rename recovered edits hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-1c82b83631a2%2C43333%2C1732071649765.meta.1732071651840.meta.temp to hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 at 1732071655098 (+2 ms)Processed 58 edits across 1 Regions in 34 ms; skipped=1; WAL=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/WALs/1c82b83631a2,43333,1732071649765-splitting/1c82b83631a2%2C43333%2C1732071649765.meta.1732071651840.meta, size=31.4 K, length=32114, corrupted=false, cancelled=false at 1732071655103 (+5 ms) 2024-11-20T03:00:55,104 DEBUG [RS_LOG_REPLAY_OPS-regionserver/1c82b83631a2:0-0 {event_type=RS_LOG_REPLAY, pid=59}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/WALs/1c82b83631a2,43333,1732071649765-splitting/1c82b83631a2%2C43333%2C1732071649765.meta.1732071651840.meta 2024-11-20T03:00:55,106 DEBUG [RS_LOG_REPLAY_OPS-regionserver/1c82b83631a2:0-0 {event_type=RS_LOG_REPLAY, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-11-20T03:00:55,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39617 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-11-20T03:00:55,115 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/WALs/1c82b83631a2,43333,1732071649765-splitting/1c82b83631a2%2C43333%2C1732071649765.meta.1732071651840.meta to hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/oldWALs 2024-11-20T03:00:55,118 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-20T03:00:55,118 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1c82b83631a2%2C43333%2C1732071649765.meta.1732071651840.meta, worker=1c82b83631a2,36133,1732071649570 in 262 msec 2024-11-20T03:00:55,121 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=1c82b83631a2,36133,1732071649570 2024-11-20T03:00:55,125 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-11-20T03:00:55,125 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 1c82b83631a2%2C43333%2C1732071649765.meta.1732071651840.meta, worker=1c82b83631a2,36133,1732071649570 in 278 msec 2024-11-20T03:00:55,127 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/WALs/1c82b83631a2,43333,1732071649765-splitting dir is empty, no logs to split. 2024-11-20T03:00:55,127 INFO [PEWorker-1 {}] master.SplitWALManager(105): 1c82b83631a2,43333,1732071649765 WAL count=0, meta=true 2024-11-20T03:00:55,127 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 1c82b83631a2,43333,1732071649765 WAL splitting is done? wals=0, meta=true 2024-11-20T03:00:55,129 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-20T03:00:55,130 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T03:00:55,131 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-20T03:00:55,280 INFO [regionserver/1c82b83631a2:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-20T03:00:55,280 INFO [regionserver/1c82b83631a2:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-20T03:00:55,281 DEBUG [1c82b83631a2:39617 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-20T03:00:55,282 DEBUG [1c82b83631a2:39617 {}] balancer.BalancerClusterState(204): Hosts are {1c82b83631a2=0} racks are {/default-rack=0} 2024-11-20T03:00:55,282 DEBUG [1c82b83631a2:39617 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-20T03:00:55,282 DEBUG [1c82b83631a2:39617 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-20T03:00:55,282 DEBUG [1c82b83631a2:39617 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-20T03:00:55,282 DEBUG [1c82b83631a2:39617 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-20T03:00:55,282 INFO [1c82b83631a2:39617 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-20T03:00:55,282 INFO [1c82b83631a2:39617 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-20T03:00:55,282 DEBUG [1c82b83631a2:39617 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-20T03:00:55,282 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=1c82b83631a2,42975,1732071649704 2024-11-20T03:00:55,284 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1c82b83631a2,42975,1732071649704, state=OPENING 2024-11-20T03:00:55,330 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39617-0x10157797d910000, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T03:00:55,330 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42975-0x10157797d910002, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T03:00:55,330 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36133-0x10157797d910001, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T03:00:55,330 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T03:00:55,330 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T03:00:55,330 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T03:00:55,330 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=1c82b83631a2,42975,1732071649704}] 2024-11-20T03:00:55,330 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T03:00:55,494 INFO [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=61}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-20T03:00:55,494 INFO [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T03:00:55,495 INFO [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=61}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-20T03:00:55,499 INFO [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=1c82b83631a2%2C42975%2C1732071649704.meta, suffix=.meta, logDir=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/WALs/1c82b83631a2,42975,1732071649704, archiveDir=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/oldWALs, maxLogs=32 2024-11-20T03:00:55,515 DEBUG [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=61}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/WALs/1c82b83631a2,42975,1732071649704/1c82b83631a2%2C42975%2C1732071649704.meta.1732071655501.meta, exclude list is [], retry=0 2024-11-20T03:00:55,518 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42923,DS-26e5314a-294f-49ed-8916-15422bf99b4b,DISK] 2024-11-20T03:00:55,518 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38873,DS-afc4754b-ca55-4ef6-8e6d-da58b90deaa8,DISK] 2024-11-20T03:00:55,518 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33951,DS-eff30b0a-4ee9-45c4-bd81-37560890eea3,DISK] 2024-11-20T03:00:55,521 INFO [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/WALs/1c82b83631a2,42975,1732071649704/1c82b83631a2%2C42975%2C1732071649704.meta.1732071655501.meta 2024-11-20T03:00:55,521 DEBUG [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42603:42603),(127.0.0.1/127.0.0.1:38101:38101),(127.0.0.1/127.0.0.1:43923:43923)] 2024-11-20T03:00:55,521 DEBUG [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-20T03:00:55,522 DEBUG [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=61}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-20T03:00:55,522 DEBUG [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-20T03:00:55,522 INFO [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-20T03:00:55,522 DEBUG [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-20T03:00:55,522 DEBUG [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T03:00:55,522 DEBUG [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-20T03:00:55,522 DEBUG [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-20T03:00:55,524 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-20T03:00:55,525 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-20T03:00:55,525 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T03:00:55,526 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T03:00:55,526 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-20T03:00:55,527 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-20T03:00:55,527 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T03:00:55,528 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T03:00:55,528 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-20T03:00:55,529 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-20T03:00:55,530 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T03:00:55,530 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T03:00:55,530 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-20T03:00:55,532 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-20T03:00:55,532 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T03:00:55,532 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T03:00:55,533 DEBUG [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-20T03:00:55,533 DEBUG [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/hbase/meta/1588230740 2024-11-20T03:00:55,536 DEBUG [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/hbase/meta/1588230740 2024-11-20T03:00:55,537 INFO [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-20T03:00:55,555 DEBUG [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5793): Applied 264, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=60, path=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-20T03:00:55,558 INFO [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=28 KB heapSize=48.31 KB 2024-11-20T03:00:55,651 DEBUG [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/hbase/meta/1588230740/.tmp/info/ec515c8c5fd742a2b210f6620d297018 is 138, key is test,bbb,1732071652347.edd7cafc43290cb2489e8ba273907acd./info:regioninfo/1732071653813/Put/seqid=0 2024-11-20T03:00:55,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33951 is added to blk_1073741866_1042 (size=34924) 2024-11-20T03:00:55,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38873 is added to blk_1073741866_1042 (size=34924) 2024-11-20T03:00:55,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42923 is added to blk_1073741866_1042 (size=34924) 2024-11-20T03:00:55,691 INFO [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=27.86 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/hbase/meta/1588230740/.tmp/info/ec515c8c5fd742a2b210f6620d297018 2024-11-20T03:00:55,774 DEBUG [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/hbase/meta/1588230740/.tmp/ns/298549bdf44b41aca20fc2de8d2394be is 43, key is default/ns:d/1732071652062/Put/seqid=0 2024-11-20T03:00:55,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42923 is added to blk_1073741867_1043 (size=5153) 2024-11-20T03:00:55,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33951 is added to blk_1073741867_1043 (size=5153) 2024-11-20T03:00:55,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38873 is added to blk_1073741867_1043 (size=5153) 2024-11-20T03:00:55,784 INFO [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/hbase/meta/1588230740/.tmp/ns/298549bdf44b41aca20fc2de8d2394be 2024-11-20T03:00:55,808 DEBUG [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/hbase/meta/1588230740/.tmp/table/be1292e752e04d778bf9631a26bf14e0 is 40, key is test/table:state/1732071653930/Put/seqid=0 2024-11-20T03:00:55,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38873 is added to blk_1073741868_1044 (size=5165) 2024-11-20T03:00:55,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33951 is added to blk_1073741868_1044 (size=5165) 2024-11-20T03:00:55,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42923 is added to blk_1073741868_1044 (size=5165) 2024-11-20T03:00:55,819 INFO [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=72 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/hbase/meta/1588230740/.tmp/table/be1292e752e04d778bf9631a26bf14e0 2024-11-20T03:00:55,829 DEBUG [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/hbase/meta/1588230740/.tmp/info/ec515c8c5fd742a2b210f6620d297018 as hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/hbase/meta/1588230740/info/ec515c8c5fd742a2b210f6620d297018 2024-11-20T03:00:55,838 INFO [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/hbase/meta/1588230740/info/ec515c8c5fd742a2b210f6620d297018, entries=260, sequenceid=60, filesize=34.1 K 2024-11-20T03:00:55,842 DEBUG [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/hbase/meta/1588230740/.tmp/ns/298549bdf44b41aca20fc2de8d2394be as hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/hbase/meta/1588230740/ns/298549bdf44b41aca20fc2de8d2394be 2024-11-20T03:00:55,852 INFO [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/hbase/meta/1588230740/ns/298549bdf44b41aca20fc2de8d2394be, entries=2, sequenceid=60, filesize=5.0 K 2024-11-20T03:00:55,853 DEBUG [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/hbase/meta/1588230740/.tmp/table/be1292e752e04d778bf9631a26bf14e0 as hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/hbase/meta/1588230740/table/be1292e752e04d778bf9631a26bf14e0 2024-11-20T03:00:55,861 INFO [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/hbase/meta/1588230740/table/be1292e752e04d778bf9631a26bf14e0, entries=2, sequenceid=60, filesize=5.0 K 2024-11-20T03:00:55,865 INFO [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~28 KB/28672, heapSize ~48.02 KB/49168, currentSize=0 B/0 for 1588230740 in 306ms, sequenceid=60, compaction requested=false; wal=null 2024-11-20T03:00:55,866 DEBUG [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-20T03:00:55,869 DEBUG [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-20T03:00:55,870 DEBUG [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-20T03:00:55,870 DEBUG [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-20T03:00:55,871 DEBUG [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-20T03:00:55,873 DEBUG [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-20T03:00:55,875 DEBUG [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/hbase/meta/1588230740/recovered.edits/60.seqid, newMaxSeqId=60, maxSeqId=1 2024-11-20T03:00:55,877 INFO [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=61; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67729733, jitterRate=0.009251669049263}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-20T03:00:55,877 DEBUG [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-20T03:00:55,877 DEBUG [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732071655522Writing region info on filesystem at 1732071655522Initializing all the Stores at 1732071655524 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732071655524Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732071655524Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732071655524Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732071655524Obtaining lock to block concurrent updates at 1732071655559 (+35 ms)Preparing flush snapshotting stores in 1588230740 at 1732071655559Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=28672, getHeapSize=49408, getOffHeapSize=0, getCellsCount=264 at 1732071655562 (+3 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732071655562Flushing 1588230740/info: creating writer at 1732071655564 (+2 ms)Flushing 1588230740/info: appending metadata at 1732071655637 (+73 ms)Flushing 1588230740/info: closing flushed file at 1732071655640 (+3 ms)Flushing 1588230740/ns: creating writer at 1732071655756 (+116 ms)Flushing 1588230740/ns: appending metadata at 1732071655773 (+17 ms)Flushing 1588230740/ns: closing flushed file at 1732071655773Flushing 1588230740/table: creating writer at 1732071655792 (+19 ms)Flushing 1588230740/table: appending metadata at 1732071655807 (+15 ms)Flushing 1588230740/table: closing flushed file at 1732071655807Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@12d1e983: reopening flushed file at 1732071655827 (+20 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1c4288c0: reopening flushed file at 1732071655840 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@428de7c8: reopening flushed file at 1732071655852 (+12 ms)Finished flush of dataSize ~28 KB/28672, heapSize ~48.02 KB/49168, currentSize=0 B/0 for 1588230740 in 306ms, sequenceid=60, compaction requested=false; wal=null at 1732071655865 (+13 ms)Cleaning up temporary data from old regions at 1732071655870 (+5 ms)Running coprocessor post-open hooks at 1732071655877 (+7 ms)Region opened successfully at 1732071655877 2024-11-20T03:00:55,878 INFO [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=61, masterSystemTime=1732071655483 2024-11-20T03:00:55,881 DEBUG [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-20T03:00:55,881 INFO [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=61}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-20T03:00:55,882 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=61, regionLocation=1c82b83631a2,42975,1732071649704 2024-11-20T03:00:55,883 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1c82b83631a2,42975,1732071649704, state=OPEN 2024-11-20T03:00:55,919 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39617 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=1c82b83631a2,42975,1732071649704, table=hbase:meta, region=1588230740. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-20T03:00:55,940 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42975-0x10157797d910002, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T03:00:55,940 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36133-0x10157797d910001, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T03:00:55,940 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39617-0x10157797d910000, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T03:00:55,940 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=61, ppid=60, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=1c82b83631a2,42975,1732071649704 2024-11-20T03:00:55,940 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T03:00:55,940 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T03:00:55,941 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T03:00:55,944 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-20T03:00:55,945 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1c82b83631a2,42975,1732071649704 in 610 msec 2024-11-20T03:00:55,949 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=60, resume processing ppid=57 2024-11-20T03:00:55,949 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 816 msec 2024-11-20T03:00:55,950 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): 1c82b83631a2,43333,1732071649765 had 9 regions 2024-11-20T03:00:55,951 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 1c82b83631a2,43333,1732071649765, splitWal=true, meta=true, isMeta: false 2024-11-20T03:00:55,954 INFO [PEWorker-1 {}] master.SplitWALManager(105): 1c82b83631a2,43333,1732071649765 WAL count=1, meta=false 2024-11-20T03:00:55,954 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=57, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 1c82b83631a2%2C43333%2C1732071649765.1732071651367}] 2024-11-20T03:00:55,956 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=1c82b83631a2,42975,1732071649704 2024-11-20T03:00:55,958 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 1c82b83631a2%2C43333%2C1732071649765.1732071651367, worker=1c82b83631a2,42975,1732071649704}] 2024-11-20T03:00:56,111 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=63 2024-11-20T03:00:56,129 INFO [RS_LOG_REPLAY_OPS-regionserver/1c82b83631a2:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(299): Splitting hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/WALs/1c82b83631a2,43333,1732071649765-splitting/1c82b83631a2%2C43333%2C1732071649765.1732071651367, size=2.2 K (2237bytes) 2024-11-20T03:00:56,129 INFO [RS_LOG_REPLAY_OPS-regionserver/1c82b83631a2:0-0 {event_type=RS_LOG_REPLAY, pid=63}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/WALs/1c82b83631a2,43333,1732071649765-splitting/1c82b83631a2%2C43333%2C1732071649765.1732071651367 2024-11-20T03:00:56,129 INFO [RS_LOG_REPLAY_OPS-regionserver/1c82b83631a2:0-0 {event_type=RS_LOG_REPLAY, pid=63}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/WALs/1c82b83631a2,43333,1732071649765-splitting/1c82b83631a2%2C43333%2C1732071649765.1732071651367 after 0ms 2024-11-20T03:00:56,137 INFO [RS_LOG_REPLAY_OPS-regionserver/1c82b83631a2:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(310): Open hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/WALs/1c82b83631a2,43333,1732071649765-splitting/1c82b83631a2%2C43333%2C1732071649765.1732071651367 took 9ms 2024-11-20T03:00:56,141 DEBUG [RS_LOG_REPLAY_OPS-regionserver/1c82b83631a2:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for bf73c409fe50a78c430f016d1431abfb: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-20T03:00:56,142 DEBUG [RS_LOG_REPLAY_OPS-regionserver/1c82b83631a2:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 8a818d7f8283ee7f2b5fae844b64a8ee: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-20T03:00:56,144 DEBUG [RS_LOG_REPLAY_OPS-regionserver/1c82b83631a2:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for fc28de84807b58cb0d6ceaa73127debb: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-20T03:00:56,145 DEBUG [RS_LOG_REPLAY_OPS-regionserver/1c82b83631a2:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 4327d986b4dd716142908b96606b0ad7: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-20T03:00:56,147 DEBUG [RS_LOG_REPLAY_OPS-regionserver/1c82b83631a2:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for ccd7aea6ec14848f8c6730c88ce1c780: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-20T03:00:56,148 DEBUG [RS_LOG_REPLAY_OPS-regionserver/1c82b83631a2:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 8a06021a0c54a875234d29e87253d48a: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-20T03:00:56,151 DEBUG [RS_LOG_REPLAY_OPS-regionserver/1c82b83631a2:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 38fa6330e381ad6ba7391bf9050c289e: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-20T03:00:56,154 DEBUG [RS_LOG_REPLAY_OPS-regionserver/1c82b83631a2:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for f928be6f5d7a3631384ebd87c8f80996: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-20T03:00:56,154 DEBUG [RS_LOG_REPLAY_OPS-regionserver/1c82b83631a2:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/WALs/1c82b83631a2,43333,1732071649765-splitting/1c82b83631a2%2C43333%2C1732071649765.1732071651367 so closing down 2024-11-20T03:00:56,154 DEBUG [RS_LOG_REPLAY_OPS-regionserver/1c82b83631a2:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-20T03:00:56,154 INFO [RS_LOG_REPLAY_OPS-regionserver/1c82b83631a2:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.OutputSink(145): 3 split writer threads finished 2024-11-20T03:00:56,154 INFO [RS_LOG_REPLAY_OPS-regionserver/1c82b83631a2:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(425): Processed 8 edits across 0 Regions in 16 ms; skipped=8; WAL=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/WALs/1c82b83631a2,43333,1732071649765-splitting/1c82b83631a2%2C43333%2C1732071649765.1732071651367, size=2.2 K, length=2237, corrupted=false, cancelled=false 2024-11-20T03:00:56,154 DEBUG [RS_LOG_REPLAY_OPS-regionserver/1c82b83631a2:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(428): Completed split of hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/WALs/1c82b83631a2,43333,1732071649765-splitting/1c82b83631a2%2C43333%2C1732071649765.1732071651367, journal: Splitting hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/WALs/1c82b83631a2,43333,1732071649765-splitting/1c82b83631a2%2C43333%2C1732071649765.1732071651367, size=2.2 K (2237bytes) at 1732071656129Finishing writing output for hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/WALs/1c82b83631a2,43333,1732071649765-splitting/1c82b83631a2%2C43333%2C1732071649765.1732071651367 so closing down at 1732071656154 (+25 ms)3 split writer threads finished at 1732071656154Processed 8 edits across 0 Regions in 16 ms; skipped=8; WAL=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/WALs/1c82b83631a2,43333,1732071649765-splitting/1c82b83631a2%2C43333%2C1732071649765.1732071651367, size=2.2 K, length=2237, corrupted=false, cancelled=false at 1732071656154 2024-11-20T03:00:56,154 DEBUG [RS_LOG_REPLAY_OPS-regionserver/1c82b83631a2:0-0 {event_type=RS_LOG_REPLAY, pid=63}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/WALs/1c82b83631a2,43333,1732071649765-splitting/1c82b83631a2%2C43333%2C1732071649765.1732071651367 2024-11-20T03:00:56,155 DEBUG [RS_LOG_REPLAY_OPS-regionserver/1c82b83631a2:0-0 {event_type=RS_LOG_REPLAY, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-11-20T03:00:56,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39617 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-11-20T03:00:56,160 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/WALs/1c82b83631a2,43333,1732071649765-splitting/1c82b83631a2%2C43333%2C1732071649765.1732071651367 to hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/oldWALs 2024-11-20T03:00:56,162 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-20T03:00:56,162 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1c82b83631a2%2C43333%2C1732071649765.1732071651367, worker=1c82b83631a2,42975,1732071649704 in 203 msec 2024-11-20T03:00:56,163 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=1c82b83631a2,42975,1732071649704 2024-11-20T03:00:56,166 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=57 2024-11-20T03:00:56,166 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 1c82b83631a2%2C43333%2C1732071649765.1732071651367, worker=1c82b83631a2,42975,1732071649704 in 209 msec 2024-11-20T03:00:56,168 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/WALs/1c82b83631a2,43333,1732071649765-splitting dir is empty, no logs to split. 2024-11-20T03:00:56,168 INFO [PEWorker-1 {}] master.SplitWALManager(105): 1c82b83631a2,43333,1732071649765 WAL count=0, meta=false 2024-11-20T03:00:56,169 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 1c82b83631a2,43333,1732071649765 WAL splitting is done? wals=0, meta=false 2024-11-20T03:00:56,171 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=38fa6330e381ad6ba7391bf9050c289e, ASSIGN}, {pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=bf73c409fe50a78c430f016d1431abfb, ASSIGN}, {pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ccd7aea6ec14848f8c6730c88ce1c780, ASSIGN}, {pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8a06021a0c54a875234d29e87253d48a, ASSIGN}, {pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4327d986b4dd716142908b96606b0ad7, ASSIGN}, {pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=fc28de84807b58cb0d6ceaa73127debb, ASSIGN}, {pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8a818d7f8283ee7f2b5fae844b64a8ee, ASSIGN}, {pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f928be6f5d7a3631384ebd87c8f80996, ASSIGN}] 2024-11-20T03:00:56,172 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f928be6f5d7a3631384ebd87c8f80996, ASSIGN 2024-11-20T03:00:56,173 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=f928be6f5d7a3631384ebd87c8f80996, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-20T03:00:56,174 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8a818d7f8283ee7f2b5fae844b64a8ee, ASSIGN 2024-11-20T03:00:56,175 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=8a818d7f8283ee7f2b5fae844b64a8ee, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-20T03:00:56,176 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=fc28de84807b58cb0d6ceaa73127debb, ASSIGN 2024-11-20T03:00:56,176 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=fc28de84807b58cb0d6ceaa73127debb, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-20T03:00:56,177 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4327d986b4dd716142908b96606b0ad7, ASSIGN 2024-11-20T03:00:56,178 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=4327d986b4dd716142908b96606b0ad7, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-20T03:00:56,179 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8a06021a0c54a875234d29e87253d48a, ASSIGN 2024-11-20T03:00:56,179 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=8a06021a0c54a875234d29e87253d48a, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-20T03:00:56,180 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ccd7aea6ec14848f8c6730c88ce1c780, ASSIGN 2024-11-20T03:00:56,181 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=ccd7aea6ec14848f8c6730c88ce1c780, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-20T03:00:56,181 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=bf73c409fe50a78c430f016d1431abfb, ASSIGN 2024-11-20T03:00:56,182 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=bf73c409fe50a78c430f016d1431abfb, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-20T03:00:56,183 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=38fa6330e381ad6ba7391bf9050c289e, ASSIGN 2024-11-20T03:00:56,184 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=38fa6330e381ad6ba7391bf9050c289e, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-20T03:00:56,324 DEBUG [1c82b83631a2:39617 {}] balancer.BalancerClusterState(204): Hosts are {1c82b83631a2=0} racks are {/default-rack=0} 2024-11-20T03:00:56,324 DEBUG [1c82b83631a2:39617 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-20T03:00:56,324 DEBUG [1c82b83631a2:39617 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-20T03:00:56,324 DEBUG [1c82b83631a2:39617 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-20T03:00:56,324 DEBUG [1c82b83631a2:39617 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-20T03:00:56,324 INFO [1c82b83631a2:39617 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-20T03:00:56,324 INFO [1c82b83631a2:39617 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-20T03:00:56,324 DEBUG [1c82b83631a2:39617 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-20T03:00:56,325 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=65 updating hbase:meta row=bf73c409fe50a78c430f016d1431abfb, regionState=OPENING, regionLocation=1c82b83631a2,42975,1732071649704 2024-11-20T03:00:56,325 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=70 updating hbase:meta row=8a818d7f8283ee7f2b5fae844b64a8ee, regionState=OPENING, regionLocation=1c82b83631a2,42975,1732071649704 2024-11-20T03:00:56,326 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=38fa6330e381ad6ba7391bf9050c289e, regionState=OPENING, regionLocation=1c82b83631a2,42975,1732071649704 2024-11-20T03:00:56,327 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=69 updating hbase:meta row=fc28de84807b58cb0d6ceaa73127debb, regionState=OPENING, regionLocation=1c82b83631a2,42975,1732071649704 2024-11-20T03:00:56,326 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 1c82b83631a2:43333 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 1c82b83631a2/172.17.0.2:43333 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T03:00:56,327 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=67 updating hbase:meta row=8a06021a0c54a875234d29e87253d48a, regionState=OPENING, regionLocation=1c82b83631a2,36133,1732071649570 2024-11-20T03:00:56,328 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=68 updating hbase:meta row=4327d986b4dd716142908b96606b0ad7, regionState=OPENING, regionLocation=1c82b83631a2,36133,1732071649570 2024-11-20T03:00:56,328 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=71 updating hbase:meta row=f928be6f5d7a3631384ebd87c8f80996, regionState=OPENING, regionLocation=1c82b83631a2,36133,1732071649570 2024-11-20T03:00:56,328 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=1c82b83631a2,43333,1732071649765, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=1c82b83631a2,43333,1732071649765, seqNum=-1, error=java.net.ConnectException: Call to address=1c82b83631a2:43333 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 1c82b83631a2/172.17.0.2:43333 2024-11-20T03:00:56,329 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=66 updating hbase:meta row=ccd7aea6ec14848f8c6730c88ce1c780, regionState=OPENING, regionLocation=1c82b83631a2,36133,1732071649570 2024-11-20T03:00:56,329 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=1c82b83631a2,43333,1732071649765, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-20T03:00:56,329 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=1c82b83631a2,43333,1732071649765, seqNum=-1 from cache 2024-11-20T03:00:56,330 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=1c82b83631a2,43333,1732071649765, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=1c82b83631a2:43333 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 1c82b83631a2/172.17.0.2:43333 2024-11-20T03:00:56,331 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.FailedServers(52): Added failed server with address 1c82b83631a2:43333 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 1c82b83631a2/172.17.0.2:43333 2024-11-20T03:00:56,331 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 1c82b83631a2:43333 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 1c82b83631a2/172.17.0.2:43333 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T03:00:56,332 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=1c82b83631a2,43333,1732071649765, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=1c82b83631a2:43333 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 1c82b83631a2/172.17.0.2:43333 2024-11-20T03:00:56,332 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=1c82b83631a2,43333,1732071649765, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=1c82b83631a2:43333 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 1c82b83631a2/172.17.0.2:43333 2024-11-20T03:00:56,332 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=1c82b83631a2,43333,1732071649765, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=1c82b83631a2:43333 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 1c82b83631a2/172.17.0.2:43333 2024-11-20T03:00:56,332 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=1c82b83631a2,43333,1732071649765, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=1c82b83631a2:43333 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 1c82b83631a2/172.17.0.2:43333 2024-11-20T03:00:56,332 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=1c82b83631a2,43333,1732071649765, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=1c82b83631a2:43333 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 1c82b83631a2/172.17.0.2:43333 2024-11-20T03:00:56,332 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=1c82b83631a2,43333,1732071649765, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=1c82b83631a2:43333 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 1c82b83631a2/172.17.0.2:43333 2024-11-20T03:00:56,333 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.FailedServers(52): Added failed server with address 1c82b83631a2:43333 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 1c82b83631a2/172.17.0.2:43333 2024-11-20T03:00:56,439 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T03:00:56,439 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1c82b83631a2,42975,1732071649704, seqNum=-1] 2024-11-20T03:00:56,439 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T03:00:56,442 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45993, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T03:00:56,445 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=bf73c409fe50a78c430f016d1431abfb, ASSIGN because future has completed 2024-11-20T03:00:56,446 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=72, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure bf73c409fe50a78c430f016d1431abfb, server=1c82b83631a2,42975,1732071649704}] 2024-11-20T03:00:56,447 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=8a818d7f8283ee7f2b5fae844b64a8ee, ASSIGN because future has completed 2024-11-20T03:00:56,448 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=38fa6330e381ad6ba7391bf9050c289e, ASSIGN because future has completed 2024-11-20T03:00:56,450 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=fc28de84807b58cb0d6ceaa73127debb, ASSIGN because future has completed 2024-11-20T03:00:56,450 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure fc28de84807b58cb0d6ceaa73127debb, server=1c82b83631a2,42975,1732071649704}] 2024-11-20T03:00:56,451 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=8a06021a0c54a875234d29e87253d48a, ASSIGN because future has completed 2024-11-20T03:00:56,452 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=4327d986b4dd716142908b96606b0ad7, ASSIGN because future has completed 2024-11-20T03:00:56,453 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=f928be6f5d7a3631384ebd87c8f80996, ASSIGN because future has completed 2024-11-20T03:00:56,454 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=ccd7aea6ec14848f8c6730c88ce1c780, ASSIGN because future has completed 2024-11-20T03:00:56,456 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=74, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure ccd7aea6ec14848f8c6730c88ce1c780, server=1c82b83631a2,36133,1732071649570}] 2024-11-20T03:00:56,459 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure f928be6f5d7a3631384ebd87c8f80996, server=1c82b83631a2,36133,1732071649570}] 2024-11-20T03:00:56,463 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4327d986b4dd716142908b96606b0ad7, server=1c82b83631a2,36133,1732071649570}] 2024-11-20T03:00:56,467 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8a06021a0c54a875234d29e87253d48a, server=1c82b83631a2,36133,1732071649570}] 2024-11-20T03:00:56,471 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=78, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 38fa6330e381ad6ba7391bf9050c289e, server=1c82b83631a2,42975,1732071649704}] 2024-11-20T03:00:56,476 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8a818d7f8283ee7f2b5fae844b64a8ee, server=1c82b83631a2,42975,1732071649704}] 2024-11-20T03:00:56,602 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] handler.AssignRegionHandler(132): Open test,zzz,1732071652347.bf73c409fe50a78c430f016d1431abfb. 2024-11-20T03:00:56,602 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(7752): Opening region: {ENCODED => bf73c409fe50a78c430f016d1431abfb, NAME => 'test,zzz,1732071652347.bf73c409fe50a78c430f016d1431abfb.', STARTKEY => 'zzz', ENDKEY => ''} 2024-11-20T03:00:56,603 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test bf73c409fe50a78c430f016d1431abfb 2024-11-20T03:00:56,603 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(898): Instantiated test,zzz,1732071652347.bf73c409fe50a78c430f016d1431abfb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T03:00:56,603 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(7794): checking encryption for bf73c409fe50a78c430f016d1431abfb 2024-11-20T03:00:56,603 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(7797): checking classloading for bf73c409fe50a78c430f016d1431abfb 2024-11-20T03:00:56,605 INFO [StoreOpener-bf73c409fe50a78c430f016d1431abfb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region bf73c409fe50a78c430f016d1431abfb 2024-11-20T03:00:56,606 INFO [StoreOpener-bf73c409fe50a78c430f016d1431abfb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bf73c409fe50a78c430f016d1431abfb columnFamilyName family 2024-11-20T03:00:56,606 DEBUG [StoreOpener-bf73c409fe50a78c430f016d1431abfb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T03:00:56,606 INFO [StoreOpener-bf73c409fe50a78c430f016d1431abfb-1 {}] regionserver.HStore(327): Store=bf73c409fe50a78c430f016d1431abfb/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T03:00:56,607 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1038): replaying wal for bf73c409fe50a78c430f016d1431abfb 2024-11-20T03:00:56,608 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/default/test/bf73c409fe50a78c430f016d1431abfb 2024-11-20T03:00:56,609 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/default/test/bf73c409fe50a78c430f016d1431abfb 2024-11-20T03:00:56,610 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1048): stopping wal replay for bf73c409fe50a78c430f016d1431abfb 2024-11-20T03:00:56,610 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1060): Cleaning up temporary data for bf73c409fe50a78c430f016d1431abfb 2024-11-20T03:00:56,611 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] handler.AssignRegionHandler(132): Open test,ccc,1732071652347.ccd7aea6ec14848f8c6730c88ce1c780. 2024-11-20T03:00:56,611 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(7752): Opening region: {ENCODED => ccd7aea6ec14848f8c6730c88ce1c780, NAME => 'test,ccc,1732071652347.ccd7aea6ec14848f8c6730c88ce1c780.', STARTKEY => 'ccc', ENDKEY => 'ddd'} 2024-11-20T03:00:56,611 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test ccd7aea6ec14848f8c6730c88ce1c780 2024-11-20T03:00:56,612 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(898): Instantiated test,ccc,1732071652347.ccd7aea6ec14848f8c6730c88ce1c780.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T03:00:56,612 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(7794): checking encryption for ccd7aea6ec14848f8c6730c88ce1c780 2024-11-20T03:00:56,612 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(7797): checking classloading for ccd7aea6ec14848f8c6730c88ce1c780 2024-11-20T03:00:56,612 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1093): writing seq id for bf73c409fe50a78c430f016d1431abfb 2024-11-20T03:00:56,613 INFO [StoreOpener-ccd7aea6ec14848f8c6730c88ce1c780-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region ccd7aea6ec14848f8c6730c88ce1c780 2024-11-20T03:00:56,613 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1114): Opened bf73c409fe50a78c430f016d1431abfb; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71306553, jitterRate=0.06255044043064117}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T03:00:56,614 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1122): Running coprocessor post-open hooks for bf73c409fe50a78c430f016d1431abfb 2024-11-20T03:00:56,614 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1006): Region open journal for bf73c409fe50a78c430f016d1431abfb: Running coprocessor pre-open hook at 1732071656603Writing region info on filesystem at 1732071656603Initializing all the Stores at 1732071656604 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732071656604Cleaning up temporary data from old regions at 1732071656610 (+6 ms)Running coprocessor post-open hooks at 1732071656614 (+4 ms)Region opened successfully at 1732071656614 2024-11-20T03:00:56,614 INFO [StoreOpener-ccd7aea6ec14848f8c6730c88ce1c780-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ccd7aea6ec14848f8c6730c88ce1c780 columnFamilyName family 2024-11-20T03:00:56,614 DEBUG [StoreOpener-ccd7aea6ec14848f8c6730c88ce1c780-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T03:00:56,615 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegionServer(2236): Post open deploy tasks for test,zzz,1732071652347.bf73c409fe50a78c430f016d1431abfb., pid=72, masterSystemTime=1732071656598 2024-11-20T03:00:56,615 INFO [StoreOpener-ccd7aea6ec14848f8c6730c88ce1c780-1 {}] regionserver.HStore(327): Store=ccd7aea6ec14848f8c6730c88ce1c780/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T03:00:56,615 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1038): replaying wal for ccd7aea6ec14848f8c6730c88ce1c780 2024-11-20T03:00:56,616 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/default/test/ccd7aea6ec14848f8c6730c88ce1c780 2024-11-20T03:00:56,618 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegionServer(2266): Finished post open deploy task for test,zzz,1732071652347.bf73c409fe50a78c430f016d1431abfb. 2024-11-20T03:00:56,618 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] handler.AssignRegionHandler(153): Opened test,zzz,1732071652347.bf73c409fe50a78c430f016d1431abfb. 2024-11-20T03:00:56,618 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] handler.AssignRegionHandler(132): Open test,mmm,1732071652347.38fa6330e381ad6ba7391bf9050c289e. 2024-11-20T03:00:56,618 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/default/test/ccd7aea6ec14848f8c6730c88ce1c780 2024-11-20T03:00:56,618 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7752): Opening region: {ENCODED => 38fa6330e381ad6ba7391bf9050c289e, NAME => 'test,mmm,1732071652347.38fa6330e381ad6ba7391bf9050c289e.', STARTKEY => 'mmm', ENDKEY => 'nnn'} 2024-11-20T03:00:56,619 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1048): stopping wal replay for ccd7aea6ec14848f8c6730c88ce1c780 2024-11-20T03:00:56,619 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 38fa6330e381ad6ba7391bf9050c289e 2024-11-20T03:00:56,619 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1060): Cleaning up temporary data for ccd7aea6ec14848f8c6730c88ce1c780 2024-11-20T03:00:56,619 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(898): Instantiated test,mmm,1732071652347.38fa6330e381ad6ba7391bf9050c289e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T03:00:56,619 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7794): checking encryption for 38fa6330e381ad6ba7391bf9050c289e 2024-11-20T03:00:56,619 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7797): checking classloading for 38fa6330e381ad6ba7391bf9050c289e 2024-11-20T03:00:56,621 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1093): writing seq id for ccd7aea6ec14848f8c6730c88ce1c780 2024-11-20T03:00:56,621 INFO [StoreOpener-38fa6330e381ad6ba7391bf9050c289e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 38fa6330e381ad6ba7391bf9050c289e 2024-11-20T03:00:56,621 WARN [PEWorker-1 {}] procedure2.ProcedureTestingUtility(201): Set Kill before store update in rollback to: true 2024-11-20T03:00:56,621 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=72, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf73c409fe50a78c430f016d1431abfb, server=1c82b83631a2,42975,1732071649704 2024-11-20T03:00:56,622 INFO [StoreOpener-38fa6330e381ad6ba7391bf9050c289e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 38fa6330e381ad6ba7391bf9050c289e columnFamilyName family 2024-11-20T03:00:56,622 DEBUG [StoreOpener-38fa6330e381ad6ba7391bf9050c289e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T03:00:56,622 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1114): Opened ccd7aea6ec14848f8c6730c88ce1c780; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61807974, jitterRate=-0.0789894163608551}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T03:00:56,622 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ccd7aea6ec14848f8c6730c88ce1c780 2024-11-20T03:00:56,622 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1006): Region open journal for ccd7aea6ec14848f8c6730c88ce1c780: Running coprocessor pre-open hook at 1732071656612Writing region info on filesystem at 1732071656612Initializing all the Stores at 1732071656613 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732071656613Cleaning up temporary data from old regions at 1732071656619 (+6 ms)Running coprocessor post-open hooks at 1732071656622 (+3 ms)Region opened successfully at 1732071656622 2024-11-20T03:00:56,621 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1844): CODE-BUG: Uncaught runtime exception: pid=72, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf73c409fe50a78c430f016d1431abfb, server=1c82b83631a2,42975,1732071649704 java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T03:00:56,623 INFO [StoreOpener-38fa6330e381ad6ba7391bf9050c289e-1 {}] regionserver.HStore(327): Store=38fa6330e381ad6ba7391bf9050c289e/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T03:00:56,623 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1038): replaying wal for 38fa6330e381ad6ba7391bf9050c289e 2024-11-20T03:00:56,623 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ccc,1732071652347.ccd7aea6ec14848f8c6730c88ce1c780., pid=74, masterSystemTime=1732071656607 2024-11-20T03:00:56,624 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/default/test/38fa6330e381ad6ba7391bf9050c289e 2024-11-20T03:00:56,626 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ccc,1732071652347.ccd7aea6ec14848f8c6730c88ce1c780. 2024-11-20T03:00:56,626 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/default/test/38fa6330e381ad6ba7391bf9050c289e 2024-11-20T03:00:56,626 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] handler.AssignRegionHandler(153): Opened test,ccc,1732071652347.ccd7aea6ec14848f8c6730c88ce1c780. 2024-11-20T03:00:56,627 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] handler.AssignRegionHandler(132): Open test,ooo,1732071652347.4327d986b4dd716142908b96606b0ad7. 2024-11-20T03:00:56,627 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7752): Opening region: {ENCODED => 4327d986b4dd716142908b96606b0ad7, NAME => 'test,ooo,1732071652347.4327d986b4dd716142908b96606b0ad7.', STARTKEY => 'ooo', ENDKEY => 'ppp'} 2024-11-20T03:00:56,627 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1048): stopping wal replay for 38fa6330e381ad6ba7391bf9050c289e 2024-11-20T03:00:56,627 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1060): Cleaning up temporary data for 38fa6330e381ad6ba7391bf9050c289e 2024-11-20T03:00:56,627 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 4327d986b4dd716142908b96606b0ad7 2024-11-20T03:00:56,627 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(898): Instantiated test,ooo,1732071652347.4327d986b4dd716142908b96606b0ad7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T03:00:56,627 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7794): checking encryption for 4327d986b4dd716142908b96606b0ad7 2024-11-20T03:00:56,627 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7797): checking classloading for 4327d986b4dd716142908b96606b0ad7 2024-11-20T03:00:56,629 INFO [StoreOpener-4327d986b4dd716142908b96606b0ad7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 4327d986b4dd716142908b96606b0ad7 2024-11-20T03:00:56,629 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1093): writing seq id for 38fa6330e381ad6ba7391bf9050c289e 2024-11-20T03:00:56,630 INFO [StoreOpener-4327d986b4dd716142908b96606b0ad7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4327d986b4dd716142908b96606b0ad7 columnFamilyName family 2024-11-20T03:00:56,630 DEBUG [StoreOpener-4327d986b4dd716142908b96606b0ad7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T03:00:56,631 INFO [StoreOpener-4327d986b4dd716142908b96606b0ad7-1 {}] regionserver.HStore(327): Store=4327d986b4dd716142908b96606b0ad7/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T03:00:56,631 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1038): replaying wal for 4327d986b4dd716142908b96606b0ad7 2024-11-20T03:00:56,631 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1114): Opened 38fa6330e381ad6ba7391bf9050c289e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59898489, jitterRate=-0.1074429601430893}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T03:00:56,632 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 38fa6330e381ad6ba7391bf9050c289e 2024-11-20T03:00:56,632 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1006): Region open journal for 38fa6330e381ad6ba7391bf9050c289e: Running coprocessor pre-open hook at 1732071656619Writing region info on filesystem at 1732071656619Initializing all the Stores at 1732071656620 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732071656621 (+1 ms)Cleaning up temporary data from old regions at 1732071656627 (+6 ms)Running coprocessor post-open hooks at 1732071656632 (+5 ms)Region opened successfully at 1732071656632 2024-11-20T03:00:56,632 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/default/test/4327d986b4dd716142908b96606b0ad7 2024-11-20T03:00:56,633 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionServer(2236): Post open deploy tasks for test,mmm,1732071652347.38fa6330e381ad6ba7391bf9050c289e., pid=78, masterSystemTime=1732071656598 2024-11-20T03:00:56,632 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=true, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf73c409fe50a78c430f016d1431abfb, server=1c82b83631a2,42975,1732071649704:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1c82b83631a2,43333,1732071649765, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1845) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 4 more 2024-11-20T03:00:56,635 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/default/test/4327d986b4dd716142908b96606b0ad7 2024-11-20T03:00:56,635 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-20T03:00:56,635 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=79, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8a818d7f8283ee7f2b5fae844b64a8ee, server=1c82b83631a2,42975,1732071649704 2024-11-20T03:00:56,635 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T03:00:56,635 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionServer(2266): Finished post open deploy task for test,mmm,1732071652347.38fa6330e381ad6ba7391bf9050c289e. 2024-11-20T03:00:56,635 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] handler.AssignRegionHandler(153): Opened test,mmm,1732071652347.38fa6330e381ad6ba7391bf9050c289e. 2024-11-20T03:00:56,635 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1048): stopping wal replay for 4327d986b4dd716142908b96606b0ad7 2024-11-20T03:00:56,635 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(132): Open test,ttt,1732071652347.8a818d7f8283ee7f2b5fae844b64a8ee. 2024-11-20T03:00:56,636 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1060): Cleaning up temporary data for 4327d986b4dd716142908b96606b0ad7 2024-11-20T03:00:56,636 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=79, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8a818d7f8283ee7f2b5fae844b64a8ee, server=1c82b83631a2,42975,1732071649704 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T03:00:56,636 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7752): Opening region: {ENCODED => 8a818d7f8283ee7f2b5fae844b64a8ee, NAME => 'test,ttt,1732071652347.8a818d7f8283ee7f2b5fae844b64a8ee.', STARTKEY => 'ttt', ENDKEY => 'uuu'} 2024-11-20T03:00:56,636 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 8a818d7f8283ee7f2b5fae844b64a8ee 2024-11-20T03:00:56,636 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(898): Instantiated test,ttt,1732071652347.8a818d7f8283ee7f2b5fae844b64a8ee.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T03:00:56,636 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7794): checking encryption for 8a818d7f8283ee7f2b5fae844b64a8ee 2024-11-20T03:00:56,636 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7797): checking classloading for 8a818d7f8283ee7f2b5fae844b64a8ee 2024-11-20T03:00:56,638 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1093): writing seq id for 4327d986b4dd716142908b96606b0ad7 2024-11-20T03:00:56,638 INFO [StoreOpener-8a818d7f8283ee7f2b5fae844b64a8ee-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 8a818d7f8283ee7f2b5fae844b64a8ee 2024-11-20T03:00:56,639 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1114): Opened 4327d986b4dd716142908b96606b0ad7; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60755302, jitterRate=-0.0946754515171051}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T03:00:56,639 INFO [StoreOpener-8a818d7f8283ee7f2b5fae844b64a8ee-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8a818d7f8283ee7f2b5fae844b64a8ee columnFamilyName family 2024-11-20T03:00:56,639 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4327d986b4dd716142908b96606b0ad7 2024-11-20T03:00:56,639 DEBUG [StoreOpener-8a818d7f8283ee7f2b5fae844b64a8ee-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T03:00:56,639 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1006): Region open journal for 4327d986b4dd716142908b96606b0ad7: Running coprocessor pre-open hook at 1732071656628Writing region info on filesystem at 1732071656628Initializing all the Stores at 1732071656628Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732071656629 (+1 ms)Cleaning up temporary data from old regions at 1732071656636 (+7 ms)Running coprocessor post-open hooks at 1732071656639 (+3 ms)Region opened successfully at 1732071656639 2024-11-20T03:00:56,639 INFO [StoreOpener-8a818d7f8283ee7f2b5fae844b64a8ee-1 {}] regionserver.HStore(327): Store=8a818d7f8283ee7f2b5fae844b64a8ee/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T03:00:56,640 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1038): replaying wal for 8a818d7f8283ee7f2b5fae844b64a8ee 2024-11-20T03:00:56,640 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ooo,1732071652347.4327d986b4dd716142908b96606b0ad7., pid=76, masterSystemTime=1732071656607 2024-11-20T03:00:56,641 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/default/test/8a818d7f8283ee7f2b5fae844b64a8ee 2024-11-20T03:00:56,643 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/default/test/8a818d7f8283ee7f2b5fae844b64a8ee 2024-11-20T03:00:56,644 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ooo,1732071652347.4327d986b4dd716142908b96606b0ad7. 2024-11-20T03:00:56,644 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] handler.AssignRegionHandler(153): Opened test,ooo,1732071652347.4327d986b4dd716142908b96606b0ad7. 2024-11-20T03:00:56,644 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(132): Open test,www,1732071652347.8a06021a0c54a875234d29e87253d48a. 2024-11-20T03:00:56,644 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1048): stopping wal replay for 8a818d7f8283ee7f2b5fae844b64a8ee 2024-11-20T03:00:56,644 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1060): Cleaning up temporary data for 8a818d7f8283ee7f2b5fae844b64a8ee 2024-11-20T03:00:56,644 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7752): Opening region: {ENCODED => 8a06021a0c54a875234d29e87253d48a, NAME => 'test,www,1732071652347.8a06021a0c54a875234d29e87253d48a.', STARTKEY => 'www', ENDKEY => 'xxx'} 2024-11-20T03:00:56,645 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 8a06021a0c54a875234d29e87253d48a 2024-11-20T03:00:56,645 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(898): Instantiated test,www,1732071652347.8a06021a0c54a875234d29e87253d48a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T03:00:56,645 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7794): checking encryption for 8a06021a0c54a875234d29e87253d48a 2024-11-20T03:00:56,645 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7797): checking classloading for 8a06021a0c54a875234d29e87253d48a 2024-11-20T03:00:56,646 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1093): writing seq id for 8a818d7f8283ee7f2b5fae844b64a8ee 2024-11-20T03:00:56,646 INFO [StoreOpener-8a06021a0c54a875234d29e87253d48a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 8a06021a0c54a875234d29e87253d48a 2024-11-20T03:00:56,647 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1114): Opened 8a818d7f8283ee7f2b5fae844b64a8ee; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71040140, jitterRate=0.05858057737350464}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T03:00:56,648 INFO [StoreOpener-8a06021a0c54a875234d29e87253d48a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8a06021a0c54a875234d29e87253d48a columnFamilyName family 2024-11-20T03:00:56,648 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8a818d7f8283ee7f2b5fae844b64a8ee 2024-11-20T03:00:56,648 DEBUG [StoreOpener-8a06021a0c54a875234d29e87253d48a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T03:00:56,648 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1006): Region open journal for 8a818d7f8283ee7f2b5fae844b64a8ee: Running coprocessor pre-open hook at 1732071656636Writing region info on filesystem at 1732071656636Initializing all the Stores at 1732071656637 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732071656637Cleaning up temporary data from old regions at 1732071656644 (+7 ms)Running coprocessor post-open hooks at 1732071656648 (+4 ms)Region opened successfully at 1732071656648 2024-11-20T03:00:56,648 INFO [StoreOpener-8a06021a0c54a875234d29e87253d48a-1 {}] regionserver.HStore(327): Store=8a06021a0c54a875234d29e87253d48a/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T03:00:56,648 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1038): replaying wal for 8a06021a0c54a875234d29e87253d48a 2024-11-20T03:00:56,649 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ttt,1732071652347.8a818d7f8283ee7f2b5fae844b64a8ee., pid=79, masterSystemTime=1732071656598 2024-11-20T03:00:56,649 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/default/test/8a06021a0c54a875234d29e87253d48a 2024-11-20T03:00:56,651 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ttt,1732071652347.8a818d7f8283ee7f2b5fae844b64a8ee. 2024-11-20T03:00:56,651 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(153): Opened test,ttt,1732071652347.8a818d7f8283ee7f2b5fae844b64a8ee. 2024-11-20T03:00:56,651 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/default/test/8a06021a0c54a875234d29e87253d48a 2024-11-20T03:00:56,651 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] handler.AssignRegionHandler(132): Open test,fff,1732071652347.fc28de84807b58cb0d6ceaa73127debb. 2024-11-20T03:00:56,652 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(7752): Opening region: {ENCODED => fc28de84807b58cb0d6ceaa73127debb, NAME => 'test,fff,1732071652347.fc28de84807b58cb0d6ceaa73127debb.', STARTKEY => 'fff', ENDKEY => 'ggg'} 2024-11-20T03:00:56,652 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1048): stopping wal replay for 8a06021a0c54a875234d29e87253d48a 2024-11-20T03:00:56,652 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1060): Cleaning up temporary data for 8a06021a0c54a875234d29e87253d48a 2024-11-20T03:00:56,652 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test fc28de84807b58cb0d6ceaa73127debb 2024-11-20T03:00:56,652 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(898): Instantiated test,fff,1732071652347.fc28de84807b58cb0d6ceaa73127debb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T03:00:56,652 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(7794): checking encryption for fc28de84807b58cb0d6ceaa73127debb 2024-11-20T03:00:56,652 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(7797): checking classloading for fc28de84807b58cb0d6ceaa73127debb 2024-11-20T03:00:56,654 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1093): writing seq id for 8a06021a0c54a875234d29e87253d48a 2024-11-20T03:00:56,654 INFO [StoreOpener-fc28de84807b58cb0d6ceaa73127debb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region fc28de84807b58cb0d6ceaa73127debb 2024-11-20T03:00:56,655 INFO [StoreOpener-fc28de84807b58cb0d6ceaa73127debb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fc28de84807b58cb0d6ceaa73127debb columnFamilyName family 2024-11-20T03:00:56,655 DEBUG [StoreOpener-fc28de84807b58cb0d6ceaa73127debb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T03:00:56,655 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1114): Opened 8a06021a0c54a875234d29e87253d48a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72774488, jitterRate=0.08442437648773193}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T03:00:56,655 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8a06021a0c54a875234d29e87253d48a 2024-11-20T03:00:56,655 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1006): Region open journal for 8a06021a0c54a875234d29e87253d48a: Running coprocessor pre-open hook at 1732071656645Writing region info on filesystem at 1732071656645Initializing all the Stores at 1732071656646 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732071656646Cleaning up temporary data from old regions at 1732071656652 (+6 ms)Running coprocessor post-open hooks at 1732071656655 (+3 ms)Region opened successfully at 1732071656655 2024-11-20T03:00:56,655 INFO [StoreOpener-fc28de84807b58cb0d6ceaa73127debb-1 {}] regionserver.HStore(327): Store=fc28de84807b58cb0d6ceaa73127debb/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T03:00:56,655 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1038): replaying wal for fc28de84807b58cb0d6ceaa73127debb 2024-11-20T03:00:56,656 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2236): Post open deploy tasks for test,www,1732071652347.8a06021a0c54a875234d29e87253d48a., pid=77, masterSystemTime=1732071656607 2024-11-20T03:00:56,656 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/default/test/fc28de84807b58cb0d6ceaa73127debb 2024-11-20T03:00:56,658 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/default/test/fc28de84807b58cb0d6ceaa73127debb 2024-11-20T03:00:56,659 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2266): Finished post open deploy task for test,www,1732071652347.8a06021a0c54a875234d29e87253d48a. 2024-11-20T03:00:56,659 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(153): Opened test,www,1732071652347.8a06021a0c54a875234d29e87253d48a. 2024-11-20T03:00:56,659 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(132): Open test,iii,1732071652347.f928be6f5d7a3631384ebd87c8f80996. 2024-11-20T03:00:56,659 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1048): stopping wal replay for fc28de84807b58cb0d6ceaa73127debb 2024-11-20T03:00:56,659 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1060): Cleaning up temporary data for fc28de84807b58cb0d6ceaa73127debb 2024-11-20T03:00:56,659 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7752): Opening region: {ENCODED => f928be6f5d7a3631384ebd87c8f80996, NAME => 'test,iii,1732071652347.f928be6f5d7a3631384ebd87c8f80996.', STARTKEY => 'iii', ENDKEY => 'jjj'} 2024-11-20T03:00:56,659 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test f928be6f5d7a3631384ebd87c8f80996 2024-11-20T03:00:56,659 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(898): Instantiated test,iii,1732071652347.f928be6f5d7a3631384ebd87c8f80996.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T03:00:56,660 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7794): checking encryption for f928be6f5d7a3631384ebd87c8f80996 2024-11-20T03:00:56,660 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7797): checking classloading for f928be6f5d7a3631384ebd87c8f80996 2024-11-20T03:00:56,661 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1093): writing seq id for fc28de84807b58cb0d6ceaa73127debb 2024-11-20T03:00:56,661 INFO [StoreOpener-f928be6f5d7a3631384ebd87c8f80996-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region f928be6f5d7a3631384ebd87c8f80996 2024-11-20T03:00:56,662 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1114): Opened fc28de84807b58cb0d6ceaa73127debb; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62263565, jitterRate=-0.07220058143138885}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T03:00:56,662 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fc28de84807b58cb0d6ceaa73127debb 2024-11-20T03:00:56,662 INFO [StoreOpener-f928be6f5d7a3631384ebd87c8f80996-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f928be6f5d7a3631384ebd87c8f80996 columnFamilyName family 2024-11-20T03:00:56,662 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1006): Region open journal for fc28de84807b58cb0d6ceaa73127debb: Running coprocessor pre-open hook at 1732071656652Writing region info on filesystem at 1732071656652Initializing all the Stores at 1732071656653 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732071656653Cleaning up temporary data from old regions at 1732071656659 (+6 ms)Running coprocessor post-open hooks at 1732071656662 (+3 ms)Region opened successfully at 1732071656662 2024-11-20T03:00:56,662 DEBUG [StoreOpener-f928be6f5d7a3631384ebd87c8f80996-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T03:00:56,663 INFO [StoreOpener-f928be6f5d7a3631384ebd87c8f80996-1 {}] regionserver.HStore(327): Store=f928be6f5d7a3631384ebd87c8f80996/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T03:00:56,663 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegionServer(2236): Post open deploy tasks for test,fff,1732071652347.fc28de84807b58cb0d6ceaa73127debb., pid=73, masterSystemTime=1732071656598 2024-11-20T03:00:56,663 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1038): replaying wal for f928be6f5d7a3631384ebd87c8f80996 2024-11-20T03:00:56,664 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/default/test/f928be6f5d7a3631384ebd87c8f80996 2024-11-20T03:00:56,666 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegionServer(2266): Finished post open deploy task for test,fff,1732071652347.fc28de84807b58cb0d6ceaa73127debb. 2024-11-20T03:00:56,666 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] handler.AssignRegionHandler(153): Opened test,fff,1732071652347.fc28de84807b58cb0d6ceaa73127debb. 2024-11-20T03:00:56,666 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/default/test/f928be6f5d7a3631384ebd87c8f80996 2024-11-20T03:00:56,666 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1048): stopping wal replay for f928be6f5d7a3631384ebd87c8f80996 2024-11-20T03:00:56,667 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1060): Cleaning up temporary data for f928be6f5d7a3631384ebd87c8f80996 2024-11-20T03:00:56,669 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1093): writing seq id for f928be6f5d7a3631384ebd87c8f80996 2024-11-20T03:00:56,670 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1114): Opened f928be6f5d7a3631384ebd87c8f80996; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71184809, jitterRate=0.060736313462257385}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T03:00:56,670 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f928be6f5d7a3631384ebd87c8f80996 2024-11-20T03:00:56,670 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1006): Region open journal for f928be6f5d7a3631384ebd87c8f80996: Running coprocessor pre-open hook at 1732071656660Writing region info on filesystem at 1732071656660Initializing all the Stores at 1732071656661 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732071656661Cleaning up temporary data from old regions at 1732071656667 (+6 ms)Running coprocessor post-open hooks at 1732071656670 (+3 ms)Region opened successfully at 1732071656670 2024-11-20T03:00:56,671 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2236): Post open deploy tasks for test,iii,1732071652347.f928be6f5d7a3631384ebd87c8f80996., pid=75, masterSystemTime=1732071656607 2024-11-20T03:00:56,675 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2266): Finished post open deploy task for test,iii,1732071652347.f928be6f5d7a3631384ebd87c8f80996. 2024-11-20T03:00:56,675 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(153): Opened test,iii,1732071652347.f928be6f5d7a3631384ebd87c8f80996. 2024-11-20T03:00:56,732 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-20T03:00:56,732 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T03:00:56,732 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T03:00:56,733 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T03:00:56,734 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-20T03:00:56,734 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-20T03:00:56,734 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-20T03:00:56,735 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T03:00:56,736 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-20T03:00:56,745 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-20T03:00:56,745 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T03:00:56,746 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=64, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T03:00:56,746 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T03:00:56,746 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T03:00:56,746 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T03:00:56,746 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T03:00:56,746 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T03:00:56,746 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T03:00:56,746 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T03:00:56,746 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T03:00:56,746 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T03:00:56,746 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T03:00:56,746 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39617 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T03:00:56,746 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T03:00:56,746 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T03:00:56,746 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T03:00:56,746 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T03:00:56,746 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T03:00:56,747 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T03:00:56,747 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T03:00:56,747 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T03:00:56,747 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T03:00:56,747 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T03:00:56,748 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T03:00:56,748 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39617 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T03:00:56,749 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-20T03:00:56,755 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf73c409fe50a78c430f016d1431abfb, server=1c82b83631a2,42975,1732071649704:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1c82b83631a2,43333,1732071649765, splitWal=true, meta=true 2024-11-20T03:00:56,755 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 1c82b83631a2%2C43333%2C1732071649765.meta.1732071651840.meta, worker=1c82b83631a2,36133,1732071649570 2024-11-20T03:00:56,755 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1c82b83631a2%2C43333%2C1732071649765.meta.1732071651840.meta, worker=1c82b83631a2,36133,1732071649570 2024-11-20T03:00:56,755 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T03:00:56,755 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1c82b83631a2,42975,1732071649704 2024-11-20T03:00:56,755 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 1c82b83631a2%2C43333%2C1732071649765.1732071651367, worker=1c82b83631a2,42975,1732071649704 2024-11-20T03:00:56,756 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1c82b83631a2%2C43333%2C1732071649765.1732071651367, worker=1c82b83631a2,42975,1732071649704 2024-11-20T03:00:56,756 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=38fa6330e381ad6ba7391bf9050c289e, ASSIGN 2024-11-20T03:00:56,756 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf73c409fe50a78c430f016d1431abfb, ASSIGN 2024-11-20T03:00:56,756 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ccd7aea6ec14848f8c6730c88ce1c780, ASSIGN 2024-11-20T03:00:56,756 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8a06021a0c54a875234d29e87253d48a, ASSIGN 2024-11-20T03:00:56,756 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4327d986b4dd716142908b96606b0ad7, ASSIGN 2024-11-20T03:00:56,756 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fc28de84807b58cb0d6ceaa73127debb, ASSIGN 2024-11-20T03:00:56,756 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8a818d7f8283ee7f2b5fae844b64a8ee, ASSIGN 2024-11-20T03:00:56,756 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f928be6f5d7a3631384ebd87c8f80996, ASSIGN 2024-11-20T03:00:56,756 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=65, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf73c409fe50a78c430f016d1431abfb, server=1c82b83631a2,42975,1732071649704:java.lang.RuntimeException: inject code bug; OpenRegionProcedure bf73c409fe50a78c430f016d1431abfb, server=1c82b83631a2,42975,1732071649704 2024-11-20T03:00:56,756 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure fc28de84807b58cb0d6ceaa73127debb, server=1c82b83631a2,42975,1732071649704 2024-11-20T03:00:56,756 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure ccd7aea6ec14848f8c6730c88ce1c780, server=1c82b83631a2,36133,1732071649570 2024-11-20T03:00:56,756 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure f928be6f5d7a3631384ebd87c8f80996, server=1c82b83631a2,36133,1732071649570 2024-11-20T03:00:56,756 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4327d986b4dd716142908b96606b0ad7, server=1c82b83631a2,36133,1732071649570 2024-11-20T03:00:56,756 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8a06021a0c54a875234d29e87253d48a, server=1c82b83631a2,36133,1732071649570 2024-11-20T03:00:56,756 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 38fa6330e381ad6ba7391bf9050c289e, server=1c82b83631a2,42975,1732071649704 2024-11-20T03:00:56,756 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8a818d7f8283ee7f2b5fae844b64a8ee, server=1c82b83631a2,42975,1732071649704 2024-11-20T03:00:56,756 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf73c409fe50a78c430f016d1431abfb, server=1c82b83631a2,42975,1732071649704:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1c82b83631a2,43333,1732071649765, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:56,757 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 1c82b83631a2%2C43333%2C1732071649765.meta.1732071651840.meta, worker=1c82b83631a2,36133,1732071649570 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:56,757 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1c82b83631a2%2C43333%2C1732071649765.meta.1732071651840.meta, worker=1c82b83631a2,36133,1732071649570 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:56,757 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:56,757 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1c82b83631a2,42975,1732071649704 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:56,757 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 1c82b83631a2%2C43333%2C1732071649765.1732071651367, worker=1c82b83631a2,42975,1732071649704 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:56,757 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1c82b83631a2%2C43333%2C1732071649765.1732071651367, worker=1c82b83631a2,42975,1732071649704 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:56,757 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=38fa6330e381ad6ba7391bf9050c289e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:56,757 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=38fa6330e381ad6ba7391bf9050c289e, ASSIGN 2024-11-20T03:00:56,757 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf73c409fe50a78c430f016d1431abfb, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:56,757 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf73c409fe50a78c430f016d1431abfb, ASSIGN 2024-11-20T03:00:56,757 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ccd7aea6ec14848f8c6730c88ce1c780, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:56,757 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ccd7aea6ec14848f8c6730c88ce1c780, ASSIGN 2024-11-20T03:00:56,757 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8a06021a0c54a875234d29e87253d48a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:56,757 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8a06021a0c54a875234d29e87253d48a, ASSIGN 2024-11-20T03:00:56,757 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4327d986b4dd716142908b96606b0ad7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:56,757 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4327d986b4dd716142908b96606b0ad7, ASSIGN 2024-11-20T03:00:56,757 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fc28de84807b58cb0d6ceaa73127debb, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:56,757 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fc28de84807b58cb0d6ceaa73127debb, ASSIGN 2024-11-20T03:00:56,757 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8a818d7f8283ee7f2b5fae844b64a8ee, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:56,758 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8a818d7f8283ee7f2b5fae844b64a8ee, ASSIGN 2024-11-20T03:00:56,758 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f928be6f5d7a3631384ebd87c8f80996, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:56,758 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f928be6f5d7a3631384ebd87c8f80996, ASSIGN 2024-11-20T03:00:56,758 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=65, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf73c409fe50a78c430f016d1431abfb, server=1c82b83631a2,42975,1732071649704:java.lang.RuntimeException: inject code bug; OpenRegionProcedure bf73c409fe50a78c430f016d1431abfb, server=1c82b83631a2,42975,1732071649704 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:56,758 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure fc28de84807b58cb0d6ceaa73127debb, server=1c82b83631a2,42975,1732071649704 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:56,758 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure ccd7aea6ec14848f8c6730c88ce1c780, server=1c82b83631a2,36133,1732071649570 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:56,758 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure f928be6f5d7a3631384ebd87c8f80996, server=1c82b83631a2,36133,1732071649570 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:56,758 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4327d986b4dd716142908b96606b0ad7, server=1c82b83631a2,36133,1732071649570 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:56,758 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8a06021a0c54a875234d29e87253d48a, server=1c82b83631a2,36133,1732071649570 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:56,758 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 38fa6330e381ad6ba7391bf9050c289e, server=1c82b83631a2,42975,1732071649704 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:56,758 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8a818d7f8283ee7f2b5fae844b64a8ee, server=1c82b83631a2,42975,1732071649704 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:56,759 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 22 msec 2024-11-20T03:00:56,761 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=61 2024-11-20T03:00:56,761 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=1c82b83631a2,42975,1732071649704, table=hbase:meta, region=1588230740 2024-11-20T03:00:56,761 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1c82b83631a2,42975,1732071649704, state=OPEN 2024-11-20T03:00:56,788 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42975-0x10157797d910002, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T03:00:56,788 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39617-0x10157797d910000, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T03:00:56,788 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36133-0x10157797d910001, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T03:00:56,788 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T03:00:56,788 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T03:00:56,788 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T03:00:56,790 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=38fa6330e381ad6ba7391bf9050c289e, ASSIGN to state=OFFLINE, location=null, table=test, region=38fa6330e381ad6ba7391bf9050c289e to restore RIT 2024-11-20T03:00:56,790 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf73c409fe50a78c430f016d1431abfb, ASSIGN to state=OFFLINE, location=null, table=test, region=bf73c409fe50a78c430f016d1431abfb to restore RIT 2024-11-20T03:00:56,790 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ccd7aea6ec14848f8c6730c88ce1c780, ASSIGN to state=OFFLINE, location=null, table=test, region=ccd7aea6ec14848f8c6730c88ce1c780 to restore RIT 2024-11-20T03:00:56,790 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8a06021a0c54a875234d29e87253d48a, ASSIGN to state=OFFLINE, location=null, table=test, region=8a06021a0c54a875234d29e87253d48a to restore RIT 2024-11-20T03:00:56,790 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4327d986b4dd716142908b96606b0ad7, ASSIGN to state=OFFLINE, location=null, table=test, region=4327d986b4dd716142908b96606b0ad7 to restore RIT 2024-11-20T03:00:56,790 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fc28de84807b58cb0d6ceaa73127debb, ASSIGN to state=OFFLINE, location=null, table=test, region=fc28de84807b58cb0d6ceaa73127debb to restore RIT 2024-11-20T03:00:56,790 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8a818d7f8283ee7f2b5fae844b64a8ee, ASSIGN to state=OFFLINE, location=null, table=test, region=8a818d7f8283ee7f2b5fae844b64a8ee to restore RIT 2024-11-20T03:00:56,790 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f928be6f5d7a3631384ebd87c8f80996, ASSIGN to state=OFFLINE, location=null, table=test, region=f928be6f5d7a3631384ebd87c8f80996 to restore RIT 2024-11-20T03:00:56,790 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732071686790 2024-11-20T03:00:56,791 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-20T03:00:56,791 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf73c409fe50a78c430f016d1431abfb, server=1c82b83631a2,42975,1732071649704:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1c82b83631a2,43333,1732071649765, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T03:00:56,792 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-20T03:00:56,794 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-20T03:00:56,794 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=78, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 38fa6330e381ad6ba7391bf9050c289e, server=1c82b83631a2,42975,1732071649704 2024-11-20T03:00:56,794 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T03:00:56,794 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=78, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 38fa6330e381ad6ba7391bf9050c289e, server=1c82b83631a2,42975,1732071649704 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T03:00:56,815 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f63a07d19bc4399768e352bd9fcf3523, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:56,816 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=edd7cafc43290cb2489e8ba273907acd, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:56,816 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ccd7aea6ec14848f8c6730c88ce1c780, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:56,816 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=63b717291dd2cfc1f752c75b18ccc26b, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:56,817 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2fe87cdaf67461ff5c508d851e418c66, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:56,817 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fc28de84807b58cb0d6ceaa73127debb, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:56,817 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3c5e70743a336dce38e44ee0f8059378, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:56,817 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2816319e15be2a1534d61304db509629, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:56,818 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f928be6f5d7a3631384ebd87c8f80996, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:56,818 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad7c736680f4a75df049e62d21f77c48, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:56,818 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=56a64fab34378d9775e0bdf29862f7e6, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:56,818 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=535a88ef9d35d9795c43261c431087a8, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:56,819 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=38fa6330e381ad6ba7391bf9050c289e, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:56,819 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fbf7ef74de7c98db6b37604b08e67282, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:56,819 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4327d986b4dd716142908b96606b0ad7, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:56,819 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ef70776250de5f4e8ad7bd24027abe63, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:56,820 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=30cc99ff17ee8d21b793ce798b4998ba, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:56,820 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a641a5878ac82d20c96fa3dff0720791, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:56,820 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=550bbfe722915166ce74fec36828c508, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:56,820 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a818d7f8283ee7f2b5fae844b64a8ee, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:56,821 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=49f9e405ee146272624d3d205a4ebf9e, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:56,821 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=45e8fc1ea8df34f038aeb00318494097, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:56,821 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a06021a0c54a875234d29e87253d48a, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:56,821 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e8e6d28d8e1806fa7d958327b87df211, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:56,821 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=295cabe34d2d3dbccccb06cf3aa5bbf6, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:56,822 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bf73c409fe50a78c430f016d1431abfb, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:56,822 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-20T03:00:56,822 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732071716822 2024-11-20T03:00:56,822 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732071776822 2024-11-20T03:00:56,822 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 31 msec 2024-11-20T03:00:56,822 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-20T03:00:56,822 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T03:00:56,822 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T03:00:56,822 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T03:00:56,822 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-20T03:00:56,822 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-20T03:00:56,823 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-20T03:00:56,823 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T03:00:56,824 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-20T03:00:56,829 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-20T03:00:56,829 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T03:00:56,829 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=64, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T03:00:56,829 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T03:00:56,829 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T03:00:56,829 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T03:00:56,829 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T03:00:56,829 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T03:00:56,829 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T03:00:56,829 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T03:00:56,829 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T03:00:56,829 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T03:00:56,829 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T03:00:56,830 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T03:00:56,830 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T03:00:56,830 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T03:00:56,830 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T03:00:56,830 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T03:00:56,830 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T03:00:56,831 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T03:00:56,831 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T03:00:56,831 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T03:00:56,831 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T03:00:56,832 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-20T03:00:56,835 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf73c409fe50a78c430f016d1431abfb, server=1c82b83631a2,42975,1732071649704:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1c82b83631a2,43333,1732071649765, splitWal=true, meta=true 2024-11-20T03:00:56,835 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 1c82b83631a2%2C43333%2C1732071649765.meta.1732071651840.meta, worker=1c82b83631a2,36133,1732071649570 2024-11-20T03:00:56,835 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1c82b83631a2%2C43333%2C1732071649765.meta.1732071651840.meta, worker=1c82b83631a2,36133,1732071649570 2024-11-20T03:00:56,835 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T03:00:56,835 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1c82b83631a2,42975,1732071649704 2024-11-20T03:00:56,835 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 1c82b83631a2%2C43333%2C1732071649765.1732071651367, worker=1c82b83631a2,42975,1732071649704 2024-11-20T03:00:56,835 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1c82b83631a2%2C43333%2C1732071649765.1732071651367, worker=1c82b83631a2,42975,1732071649704 2024-11-20T03:00:56,835 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=38fa6330e381ad6ba7391bf9050c289e, ASSIGN 2024-11-20T03:00:56,835 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf73c409fe50a78c430f016d1431abfb, ASSIGN 2024-11-20T03:00:56,835 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ccd7aea6ec14848f8c6730c88ce1c780, ASSIGN 2024-11-20T03:00:56,836 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8a06021a0c54a875234d29e87253d48a, ASSIGN 2024-11-20T03:00:56,836 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4327d986b4dd716142908b96606b0ad7, ASSIGN 2024-11-20T03:00:56,836 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fc28de84807b58cb0d6ceaa73127debb, ASSIGN 2024-11-20T03:00:56,836 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8a818d7f8283ee7f2b5fae844b64a8ee, ASSIGN 2024-11-20T03:00:56,836 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f928be6f5d7a3631384ebd87c8f80996, ASSIGN 2024-11-20T03:00:56,836 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=65, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf73c409fe50a78c430f016d1431abfb, server=1c82b83631a2,42975,1732071649704:java.lang.RuntimeException: inject code bug; OpenRegionProcedure bf73c409fe50a78c430f016d1431abfb, server=1c82b83631a2,42975,1732071649704 2024-11-20T03:00:56,836 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure fc28de84807b58cb0d6ceaa73127debb, server=1c82b83631a2,42975,1732071649704 2024-11-20T03:00:56,836 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure ccd7aea6ec14848f8c6730c88ce1c780, server=1c82b83631a2,36133,1732071649570 2024-11-20T03:00:56,836 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure f928be6f5d7a3631384ebd87c8f80996, server=1c82b83631a2,36133,1732071649570 2024-11-20T03:00:56,836 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4327d986b4dd716142908b96606b0ad7, server=1c82b83631a2,36133,1732071649570 2024-11-20T03:00:56,836 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8a06021a0c54a875234d29e87253d48a, server=1c82b83631a2,36133,1732071649570 2024-11-20T03:00:56,836 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 38fa6330e381ad6ba7391bf9050c289e, server=1c82b83631a2,42975,1732071649704 2024-11-20T03:00:56,836 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf73c409fe50a78c430f016d1431abfb, server=1c82b83631a2,42975,1732071649704:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1c82b83631a2,43333,1732071649765, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:56,836 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 1c82b83631a2%2C43333%2C1732071649765.meta.1732071651840.meta, worker=1c82b83631a2,36133,1732071649570 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:56,836 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1c82b83631a2%2C43333%2C1732071649765.meta.1732071651840.meta, worker=1c82b83631a2,36133,1732071649570 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:56,836 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:56,836 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1c82b83631a2,42975,1732071649704 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:56,836 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 1c82b83631a2%2C43333%2C1732071649765.1732071651367, worker=1c82b83631a2,42975,1732071649704 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:56,836 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1c82b83631a2%2C43333%2C1732071649765.1732071651367, worker=1c82b83631a2,42975,1732071649704 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:56,836 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=38fa6330e381ad6ba7391bf9050c289e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:56,836 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=38fa6330e381ad6ba7391bf9050c289e, ASSIGN 2024-11-20T03:00:56,836 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf73c409fe50a78c430f016d1431abfb, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:56,837 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf73c409fe50a78c430f016d1431abfb, ASSIGN 2024-11-20T03:00:56,837 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ccd7aea6ec14848f8c6730c88ce1c780, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:56,837 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ccd7aea6ec14848f8c6730c88ce1c780, ASSIGN 2024-11-20T03:00:56,837 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8a06021a0c54a875234d29e87253d48a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:56,837 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8a06021a0c54a875234d29e87253d48a, ASSIGN 2024-11-20T03:00:56,837 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4327d986b4dd716142908b96606b0ad7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:56,837 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4327d986b4dd716142908b96606b0ad7, ASSIGN 2024-11-20T03:00:56,837 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fc28de84807b58cb0d6ceaa73127debb, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:56,837 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fc28de84807b58cb0d6ceaa73127debb, ASSIGN 2024-11-20T03:00:56,837 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8a818d7f8283ee7f2b5fae844b64a8ee, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:56,837 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8a818d7f8283ee7f2b5fae844b64a8ee, ASSIGN 2024-11-20T03:00:56,837 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f928be6f5d7a3631384ebd87c8f80996, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:56,837 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f928be6f5d7a3631384ebd87c8f80996, ASSIGN 2024-11-20T03:00:56,837 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=65, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf73c409fe50a78c430f016d1431abfb, server=1c82b83631a2,42975,1732071649704:java.lang.RuntimeException: inject code bug; OpenRegionProcedure bf73c409fe50a78c430f016d1431abfb, server=1c82b83631a2,42975,1732071649704 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:56,837 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure fc28de84807b58cb0d6ceaa73127debb, server=1c82b83631a2,42975,1732071649704 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:56,837 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure ccd7aea6ec14848f8c6730c88ce1c780, server=1c82b83631a2,36133,1732071649570 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:56,838 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure f928be6f5d7a3631384ebd87c8f80996, server=1c82b83631a2,36133,1732071649570 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:56,838 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4327d986b4dd716142908b96606b0ad7, server=1c82b83631a2,36133,1732071649570 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:56,838 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8a06021a0c54a875234d29e87253d48a, server=1c82b83631a2,36133,1732071649570 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:56,838 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 38fa6330e381ad6ba7391bf9050c289e, server=1c82b83631a2,42975,1732071649704 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:56,838 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 14 msec 2024-11-20T03:00:56,840 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=61 2024-11-20T03:00:56,840 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=1c82b83631a2,42975,1732071649704, table=hbase:meta, region=1588230740 2024-11-20T03:00:56,840 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1c82b83631a2,42975,1732071649704, state=OPEN 2024-11-20T03:00:56,848 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42975-0x10157797d910002, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T03:00:56,848 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39617-0x10157797d910000, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T03:00:56,848 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36133-0x10157797d910001, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T03:00:56,849 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T03:00:56,849 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T03:00:56,849 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T03:00:56,850 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=38fa6330e381ad6ba7391bf9050c289e, ASSIGN to state=OFFLINE, location=null, table=test, region=38fa6330e381ad6ba7391bf9050c289e to restore RIT 2024-11-20T03:00:56,850 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf73c409fe50a78c430f016d1431abfb, ASSIGN to state=OFFLINE, location=null, table=test, region=bf73c409fe50a78c430f016d1431abfb to restore RIT 2024-11-20T03:00:56,850 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ccd7aea6ec14848f8c6730c88ce1c780, ASSIGN to state=OFFLINE, location=null, table=test, region=ccd7aea6ec14848f8c6730c88ce1c780 to restore RIT 2024-11-20T03:00:56,850 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8a06021a0c54a875234d29e87253d48a, ASSIGN to state=OFFLINE, location=null, table=test, region=8a06021a0c54a875234d29e87253d48a to restore RIT 2024-11-20T03:00:56,850 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4327d986b4dd716142908b96606b0ad7, ASSIGN to state=OFFLINE, location=null, table=test, region=4327d986b4dd716142908b96606b0ad7 to restore RIT 2024-11-20T03:00:56,851 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fc28de84807b58cb0d6ceaa73127debb, ASSIGN to state=OFFLINE, location=null, table=test, region=fc28de84807b58cb0d6ceaa73127debb to restore RIT 2024-11-20T03:00:56,851 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8a818d7f8283ee7f2b5fae844b64a8ee, ASSIGN to state=OFFLINE, location=null, table=test, region=8a818d7f8283ee7f2b5fae844b64a8ee to restore RIT 2024-11-20T03:00:56,851 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f928be6f5d7a3631384ebd87c8f80996, ASSIGN to state=OFFLINE, location=null, table=test, region=f928be6f5d7a3631384ebd87c8f80996 to restore RIT 2024-11-20T03:00:56,854 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732071686854 2024-11-20T03:00:56,855 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-20T03:00:56,855 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf73c409fe50a78c430f016d1431abfb, server=1c82b83631a2,42975,1732071649704:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1c82b83631a2,43333,1732071649765, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T03:00:56,856 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-20T03:00:56,860 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-20T03:00:56,860 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=77, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8a06021a0c54a875234d29e87253d48a, server=1c82b83631a2,36133,1732071649570 2024-11-20T03:00:56,860 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T03:00:56,860 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=77, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8a06021a0c54a875234d29e87253d48a, server=1c82b83631a2,36133,1732071649570 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T03:00:56,864 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f63a07d19bc4399768e352bd9fcf3523, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:56,864 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=edd7cafc43290cb2489e8ba273907acd, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:56,864 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ccd7aea6ec14848f8c6730c88ce1c780, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:56,864 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=63b717291dd2cfc1f752c75b18ccc26b, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:56,864 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2fe87cdaf67461ff5c508d851e418c66, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:56,865 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fc28de84807b58cb0d6ceaa73127debb, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:56,865 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3c5e70743a336dce38e44ee0f8059378, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:56,865 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2816319e15be2a1534d61304db509629, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:56,865 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f928be6f5d7a3631384ebd87c8f80996, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:56,865 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad7c736680f4a75df049e62d21f77c48, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:56,865 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=56a64fab34378d9775e0bdf29862f7e6, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:56,866 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=535a88ef9d35d9795c43261c431087a8, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:56,866 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=38fa6330e381ad6ba7391bf9050c289e, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:56,866 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fbf7ef74de7c98db6b37604b08e67282, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:56,866 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4327d986b4dd716142908b96606b0ad7, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:56,866 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ef70776250de5f4e8ad7bd24027abe63, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:56,866 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=30cc99ff17ee8d21b793ce798b4998ba, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:56,867 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a641a5878ac82d20c96fa3dff0720791, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:56,867 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=550bbfe722915166ce74fec36828c508, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:56,867 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a818d7f8283ee7f2b5fae844b64a8ee, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:56,867 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=49f9e405ee146272624d3d205a4ebf9e, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:56,867 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=45e8fc1ea8df34f038aeb00318494097, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:56,867 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a06021a0c54a875234d29e87253d48a, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:56,868 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e8e6d28d8e1806fa7d958327b87df211, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:56,868 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=295cabe34d2d3dbccccb06cf3aa5bbf6, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:56,868 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bf73c409fe50a78c430f016d1431abfb, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:56,868 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-20T03:00:56,868 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732071716868 2024-11-20T03:00:56,868 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732071776868 2024-11-20T03:00:56,868 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 13 msec 2024-11-20T03:00:56,868 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-20T03:00:56,868 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T03:00:56,868 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T03:00:56,868 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T03:00:56,869 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-20T03:00:56,869 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-20T03:00:56,869 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-20T03:00:56,869 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T03:00:56,870 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-20T03:00:56,873 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-20T03:00:56,873 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T03:00:56,873 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T03:00:56,873 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T03:00:56,873 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T03:00:56,873 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T03:00:56,873 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T03:00:56,873 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T03:00:56,873 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T03:00:56,874 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T03:00:56,874 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T03:00:56,874 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T03:00:56,874 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T03:00:56,874 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T03:00:56,874 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T03:00:56,874 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T03:00:56,874 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T03:00:56,874 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T03:00:56,874 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T03:00:56,874 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T03:00:56,874 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T03:00:56,874 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T03:00:56,875 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-20T03:00:56,878 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf73c409fe50a78c430f016d1431abfb, server=1c82b83631a2,42975,1732071649704:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1c82b83631a2,43333,1732071649765, splitWal=true, meta=true 2024-11-20T03:00:56,878 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 1c82b83631a2%2C43333%2C1732071649765.meta.1732071651840.meta, worker=1c82b83631a2,36133,1732071649570 2024-11-20T03:00:56,878 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1c82b83631a2%2C43333%2C1732071649765.meta.1732071651840.meta, worker=1c82b83631a2,36133,1732071649570 2024-11-20T03:00:56,878 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T03:00:56,878 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1c82b83631a2,42975,1732071649704 2024-11-20T03:00:56,878 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 1c82b83631a2%2C43333%2C1732071649765.1732071651367, worker=1c82b83631a2,42975,1732071649704 2024-11-20T03:00:56,878 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1c82b83631a2%2C43333%2C1732071649765.1732071651367, worker=1c82b83631a2,42975,1732071649704 2024-11-20T03:00:56,878 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=38fa6330e381ad6ba7391bf9050c289e, ASSIGN 2024-11-20T03:00:56,878 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf73c409fe50a78c430f016d1431abfb, ASSIGN 2024-11-20T03:00:56,878 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ccd7aea6ec14848f8c6730c88ce1c780, ASSIGN 2024-11-20T03:00:56,878 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8a06021a0c54a875234d29e87253d48a, ASSIGN 2024-11-20T03:00:56,878 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4327d986b4dd716142908b96606b0ad7, ASSIGN 2024-11-20T03:00:56,878 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fc28de84807b58cb0d6ceaa73127debb, ASSIGN 2024-11-20T03:00:56,878 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8a818d7f8283ee7f2b5fae844b64a8ee, ASSIGN 2024-11-20T03:00:56,878 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f928be6f5d7a3631384ebd87c8f80996, ASSIGN 2024-11-20T03:00:56,879 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=65, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf73c409fe50a78c430f016d1431abfb, server=1c82b83631a2,42975,1732071649704:java.lang.RuntimeException: inject code bug; OpenRegionProcedure bf73c409fe50a78c430f016d1431abfb, server=1c82b83631a2,42975,1732071649704 2024-11-20T03:00:56,879 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure fc28de84807b58cb0d6ceaa73127debb, server=1c82b83631a2,42975,1732071649704 2024-11-20T03:00:56,879 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure ccd7aea6ec14848f8c6730c88ce1c780, server=1c82b83631a2,36133,1732071649570 2024-11-20T03:00:56,879 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure f928be6f5d7a3631384ebd87c8f80996, server=1c82b83631a2,36133,1732071649570 2024-11-20T03:00:56,879 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4327d986b4dd716142908b96606b0ad7, server=1c82b83631a2,36133,1732071649570 2024-11-20T03:00:56,879 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8a06021a0c54a875234d29e87253d48a, server=1c82b83631a2,36133,1732071649570 2024-11-20T03:00:56,880 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf73c409fe50a78c430f016d1431abfb, server=1c82b83631a2,42975,1732071649704:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1c82b83631a2,43333,1732071649765, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:56,880 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 1c82b83631a2%2C43333%2C1732071649765.meta.1732071651840.meta, worker=1c82b83631a2,36133,1732071649570 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:56,880 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1c82b83631a2%2C43333%2C1732071649765.meta.1732071651840.meta, worker=1c82b83631a2,36133,1732071649570 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:56,880 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:56,880 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1c82b83631a2,42975,1732071649704 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:56,880 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 1c82b83631a2%2C43333%2C1732071649765.1732071651367, worker=1c82b83631a2,42975,1732071649704 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:56,880 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1c82b83631a2%2C43333%2C1732071649765.1732071651367, worker=1c82b83631a2,42975,1732071649704 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:56,880 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=38fa6330e381ad6ba7391bf9050c289e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:56,880 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=38fa6330e381ad6ba7391bf9050c289e, ASSIGN 2024-11-20T03:00:56,880 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf73c409fe50a78c430f016d1431abfb, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:56,880 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf73c409fe50a78c430f016d1431abfb, ASSIGN 2024-11-20T03:00:56,880 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ccd7aea6ec14848f8c6730c88ce1c780, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:56,880 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ccd7aea6ec14848f8c6730c88ce1c780, ASSIGN 2024-11-20T03:00:56,880 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8a06021a0c54a875234d29e87253d48a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:56,880 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8a06021a0c54a875234d29e87253d48a, ASSIGN 2024-11-20T03:00:56,880 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4327d986b4dd716142908b96606b0ad7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:56,880 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4327d986b4dd716142908b96606b0ad7, ASSIGN 2024-11-20T03:00:56,880 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fc28de84807b58cb0d6ceaa73127debb, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:56,881 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fc28de84807b58cb0d6ceaa73127debb, ASSIGN 2024-11-20T03:00:56,881 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8a818d7f8283ee7f2b5fae844b64a8ee, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:56,881 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8a818d7f8283ee7f2b5fae844b64a8ee, ASSIGN 2024-11-20T03:00:56,881 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f928be6f5d7a3631384ebd87c8f80996, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:56,881 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f928be6f5d7a3631384ebd87c8f80996, ASSIGN 2024-11-20T03:00:56,881 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=65, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf73c409fe50a78c430f016d1431abfb, server=1c82b83631a2,42975,1732071649704:java.lang.RuntimeException: inject code bug; OpenRegionProcedure bf73c409fe50a78c430f016d1431abfb, server=1c82b83631a2,42975,1732071649704 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:56,881 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure fc28de84807b58cb0d6ceaa73127debb, server=1c82b83631a2,42975,1732071649704 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:56,881 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure ccd7aea6ec14848f8c6730c88ce1c780, server=1c82b83631a2,36133,1732071649570 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:56,881 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure f928be6f5d7a3631384ebd87c8f80996, server=1c82b83631a2,36133,1732071649570 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:56,881 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4327d986b4dd716142908b96606b0ad7, server=1c82b83631a2,36133,1732071649570 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:56,881 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8a06021a0c54a875234d29e87253d48a, server=1c82b83631a2,36133,1732071649570 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:56,881 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 11 msec 2024-11-20T03:00:56,882 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=61 2024-11-20T03:00:56,882 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=1c82b83631a2,42975,1732071649704, table=hbase:meta, region=1588230740 2024-11-20T03:00:56,882 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1c82b83631a2,42975,1732071649704, state=OPEN 2024-11-20T03:00:56,890 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36133-0x10157797d910001, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T03:00:56,890 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T03:00:56,890 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39617 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T03:00:56,891 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39617 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T03:00:56,892 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42975-0x10157797d910002, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T03:00:56,892 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39617-0x10157797d910000, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T03:00:56,892 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T03:00:56,892 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T03:00:56,893 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=38fa6330e381ad6ba7391bf9050c289e, ASSIGN to state=OFFLINE, location=null, table=test, region=38fa6330e381ad6ba7391bf9050c289e to restore RIT 2024-11-20T03:00:56,893 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf73c409fe50a78c430f016d1431abfb, ASSIGN to state=OFFLINE, location=null, table=test, region=bf73c409fe50a78c430f016d1431abfb to restore RIT 2024-11-20T03:00:56,893 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ccd7aea6ec14848f8c6730c88ce1c780, ASSIGN to state=OFFLINE, location=null, table=test, region=ccd7aea6ec14848f8c6730c88ce1c780 to restore RIT 2024-11-20T03:00:56,893 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8a06021a0c54a875234d29e87253d48a, ASSIGN to state=OFFLINE, location=null, table=test, region=8a06021a0c54a875234d29e87253d48a to restore RIT 2024-11-20T03:00:56,893 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4327d986b4dd716142908b96606b0ad7, ASSIGN to state=OFFLINE, location=null, table=test, region=4327d986b4dd716142908b96606b0ad7 to restore RIT 2024-11-20T03:00:56,893 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fc28de84807b58cb0d6ceaa73127debb, ASSIGN to state=OFFLINE, location=null, table=test, region=fc28de84807b58cb0d6ceaa73127debb to restore RIT 2024-11-20T03:00:56,893 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8a818d7f8283ee7f2b5fae844b64a8ee, ASSIGN to state=OFFLINE, location=null, table=test, region=8a818d7f8283ee7f2b5fae844b64a8ee to restore RIT 2024-11-20T03:00:56,893 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f928be6f5d7a3631384ebd87c8f80996, ASSIGN to state=OFFLINE, location=null, table=test, region=f928be6f5d7a3631384ebd87c8f80996 to restore RIT 2024-11-20T03:00:56,894 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732071686894 2024-11-20T03:00:56,894 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-20T03:00:56,894 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf73c409fe50a78c430f016d1431abfb, server=1c82b83631a2,42975,1732071649704:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1c82b83631a2,43333,1732071649765, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T03:00:56,895 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-20T03:00:56,902 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f63a07d19bc4399768e352bd9fcf3523, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:56,902 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-20T03:00:56,902 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4327d986b4dd716142908b96606b0ad7, server=1c82b83631a2,36133,1732071649570 2024-11-20T03:00:56,902 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T03:00:56,903 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=edd7cafc43290cb2489e8ba273907acd, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:56,903 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4327d986b4dd716142908b96606b0ad7, server=1c82b83631a2,36133,1732071649570 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T03:00:56,903 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ccd7aea6ec14848f8c6730c88ce1c780, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:56,903 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=63b717291dd2cfc1f752c75b18ccc26b, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:56,903 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2fe87cdaf67461ff5c508d851e418c66, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:56,903 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fc28de84807b58cb0d6ceaa73127debb, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:56,903 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3c5e70743a336dce38e44ee0f8059378, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:56,904 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2816319e15be2a1534d61304db509629, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:56,904 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f928be6f5d7a3631384ebd87c8f80996, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:56,904 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad7c736680f4a75df049e62d21f77c48, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:56,904 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=56a64fab34378d9775e0bdf29862f7e6, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:56,904 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=535a88ef9d35d9795c43261c431087a8, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:56,904 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=38fa6330e381ad6ba7391bf9050c289e, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:56,904 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fbf7ef74de7c98db6b37604b08e67282, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:56,905 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4327d986b4dd716142908b96606b0ad7, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:56,905 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ef70776250de5f4e8ad7bd24027abe63, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:56,905 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=30cc99ff17ee8d21b793ce798b4998ba, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:56,905 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a641a5878ac82d20c96fa3dff0720791, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:56,905 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=550bbfe722915166ce74fec36828c508, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:56,905 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a818d7f8283ee7f2b5fae844b64a8ee, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:56,906 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=49f9e405ee146272624d3d205a4ebf9e, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:56,906 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=45e8fc1ea8df34f038aeb00318494097, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:56,906 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a06021a0c54a875234d29e87253d48a, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:56,906 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e8e6d28d8e1806fa7d958327b87df211, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:56,906 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=295cabe34d2d3dbccccb06cf3aa5bbf6, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:56,906 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bf73c409fe50a78c430f016d1431abfb, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:56,906 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-20T03:00:56,906 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732071716906 2024-11-20T03:00:56,906 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732071776906 2024-11-20T03:00:56,906 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 12 msec 2024-11-20T03:00:56,907 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-20T03:00:56,907 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T03:00:56,907 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T03:00:56,907 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T03:00:56,907 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-20T03:00:56,907 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-20T03:00:56,907 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-20T03:00:56,907 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T03:00:56,908 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-20T03:00:56,912 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-20T03:00:56,912 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T03:00:56,912 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T03:00:56,912 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T03:00:56,912 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T03:00:56,912 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T03:00:56,912 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T03:00:56,912 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T03:00:56,912 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T03:00:56,912 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T03:00:56,912 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T03:00:56,912 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T03:00:56,912 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T03:00:56,912 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T03:00:56,912 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T03:00:56,912 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T03:00:56,912 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T03:00:56,912 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T03:00:56,912 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T03:00:56,912 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T03:00:56,912 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T03:00:56,913 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-20T03:00:56,915 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf73c409fe50a78c430f016d1431abfb, server=1c82b83631a2,42975,1732071649704:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1c82b83631a2,43333,1732071649765, splitWal=true, meta=true 2024-11-20T03:00:56,915 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 1c82b83631a2%2C43333%2C1732071649765.meta.1732071651840.meta, worker=1c82b83631a2,36133,1732071649570 2024-11-20T03:00:56,915 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1c82b83631a2%2C43333%2C1732071649765.meta.1732071651840.meta, worker=1c82b83631a2,36133,1732071649570 2024-11-20T03:00:56,916 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T03:00:56,916 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1c82b83631a2,42975,1732071649704 2024-11-20T03:00:56,916 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 1c82b83631a2%2C43333%2C1732071649765.1732071651367, worker=1c82b83631a2,42975,1732071649704 2024-11-20T03:00:56,916 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1c82b83631a2%2C43333%2C1732071649765.1732071651367, worker=1c82b83631a2,42975,1732071649704 2024-11-20T03:00:56,916 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=38fa6330e381ad6ba7391bf9050c289e, ASSIGN 2024-11-20T03:00:56,916 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf73c409fe50a78c430f016d1431abfb, ASSIGN 2024-11-20T03:00:56,916 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ccd7aea6ec14848f8c6730c88ce1c780, ASSIGN 2024-11-20T03:00:56,916 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8a06021a0c54a875234d29e87253d48a, ASSIGN 2024-11-20T03:00:56,916 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4327d986b4dd716142908b96606b0ad7, ASSIGN 2024-11-20T03:00:56,916 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fc28de84807b58cb0d6ceaa73127debb, ASSIGN 2024-11-20T03:00:56,916 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8a818d7f8283ee7f2b5fae844b64a8ee, ASSIGN 2024-11-20T03:00:56,916 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f928be6f5d7a3631384ebd87c8f80996, ASSIGN 2024-11-20T03:00:56,916 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=65, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf73c409fe50a78c430f016d1431abfb, server=1c82b83631a2,42975,1732071649704:java.lang.RuntimeException: inject code bug; OpenRegionProcedure bf73c409fe50a78c430f016d1431abfb, server=1c82b83631a2,42975,1732071649704 2024-11-20T03:00:56,916 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure fc28de84807b58cb0d6ceaa73127debb, server=1c82b83631a2,42975,1732071649704 2024-11-20T03:00:56,916 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure ccd7aea6ec14848f8c6730c88ce1c780, server=1c82b83631a2,36133,1732071649570 2024-11-20T03:00:56,916 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure f928be6f5d7a3631384ebd87c8f80996, server=1c82b83631a2,36133,1732071649570 2024-11-20T03:00:56,916 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4327d986b4dd716142908b96606b0ad7, server=1c82b83631a2,36133,1732071649570 2024-11-20T03:00:56,916 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf73c409fe50a78c430f016d1431abfb, server=1c82b83631a2,42975,1732071649704:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1c82b83631a2,43333,1732071649765, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:56,916 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 1c82b83631a2%2C43333%2C1732071649765.meta.1732071651840.meta, worker=1c82b83631a2,36133,1732071649570 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:56,916 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1c82b83631a2%2C43333%2C1732071649765.meta.1732071651840.meta, worker=1c82b83631a2,36133,1732071649570 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:56,916 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:56,916 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1c82b83631a2,42975,1732071649704 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:56,916 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 1c82b83631a2%2C43333%2C1732071649765.1732071651367, worker=1c82b83631a2,42975,1732071649704 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:56,916 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1c82b83631a2%2C43333%2C1732071649765.1732071651367, worker=1c82b83631a2,42975,1732071649704 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:56,916 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=38fa6330e381ad6ba7391bf9050c289e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:56,916 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=38fa6330e381ad6ba7391bf9050c289e, ASSIGN 2024-11-20T03:00:56,916 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf73c409fe50a78c430f016d1431abfb, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:56,917 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf73c409fe50a78c430f016d1431abfb, ASSIGN 2024-11-20T03:00:56,917 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ccd7aea6ec14848f8c6730c88ce1c780, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:56,917 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ccd7aea6ec14848f8c6730c88ce1c780, ASSIGN 2024-11-20T03:00:56,917 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8a06021a0c54a875234d29e87253d48a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:56,917 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8a06021a0c54a875234d29e87253d48a, ASSIGN 2024-11-20T03:00:56,917 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4327d986b4dd716142908b96606b0ad7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:56,917 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4327d986b4dd716142908b96606b0ad7, ASSIGN 2024-11-20T03:00:56,917 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fc28de84807b58cb0d6ceaa73127debb, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:56,917 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fc28de84807b58cb0d6ceaa73127debb, ASSIGN 2024-11-20T03:00:56,917 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8a818d7f8283ee7f2b5fae844b64a8ee, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:56,917 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8a818d7f8283ee7f2b5fae844b64a8ee, ASSIGN 2024-11-20T03:00:56,917 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f928be6f5d7a3631384ebd87c8f80996, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:56,917 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f928be6f5d7a3631384ebd87c8f80996, ASSIGN 2024-11-20T03:00:56,917 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=65, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf73c409fe50a78c430f016d1431abfb, server=1c82b83631a2,42975,1732071649704:java.lang.RuntimeException: inject code bug; OpenRegionProcedure bf73c409fe50a78c430f016d1431abfb, server=1c82b83631a2,42975,1732071649704 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:56,917 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure fc28de84807b58cb0d6ceaa73127debb, server=1c82b83631a2,42975,1732071649704 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:56,917 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure ccd7aea6ec14848f8c6730c88ce1c780, server=1c82b83631a2,36133,1732071649570 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:56,917 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure f928be6f5d7a3631384ebd87c8f80996, server=1c82b83631a2,36133,1732071649570 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:56,917 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4327d986b4dd716142908b96606b0ad7, server=1c82b83631a2,36133,1732071649570 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:56,917 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 9 msec 2024-11-20T03:00:56,919 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=61 2024-11-20T03:00:56,919 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=1c82b83631a2,42975,1732071649704, table=hbase:meta, region=1588230740 2024-11-20T03:00:56,919 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1c82b83631a2,42975,1732071649704, state=OPEN 2024-11-20T03:00:56,995 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39617 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T03:00:56,995 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39617 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T03:00:57,019 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42975-0x10157797d910002, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T03:00:57,019 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39617-0x10157797d910000, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T03:00:57,019 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36133-0x10157797d910001, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T03:00:57,019 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T03:00:57,019 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T03:00:57,019 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T03:00:57,020 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=38fa6330e381ad6ba7391bf9050c289e, ASSIGN to state=OFFLINE, location=null, table=test, region=38fa6330e381ad6ba7391bf9050c289e to restore RIT 2024-11-20T03:00:57,020 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf73c409fe50a78c430f016d1431abfb, ASSIGN to state=OFFLINE, location=null, table=test, region=bf73c409fe50a78c430f016d1431abfb to restore RIT 2024-11-20T03:00:57,020 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ccd7aea6ec14848f8c6730c88ce1c780, ASSIGN to state=OFFLINE, location=null, table=test, region=ccd7aea6ec14848f8c6730c88ce1c780 to restore RIT 2024-11-20T03:00:57,020 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8a06021a0c54a875234d29e87253d48a, ASSIGN to state=OFFLINE, location=null, table=test, region=8a06021a0c54a875234d29e87253d48a to restore RIT 2024-11-20T03:00:57,020 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4327d986b4dd716142908b96606b0ad7, ASSIGN to state=OFFLINE, location=null, table=test, region=4327d986b4dd716142908b96606b0ad7 to restore RIT 2024-11-20T03:00:57,020 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fc28de84807b58cb0d6ceaa73127debb, ASSIGN to state=OFFLINE, location=null, table=test, region=fc28de84807b58cb0d6ceaa73127debb to restore RIT 2024-11-20T03:00:57,021 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8a818d7f8283ee7f2b5fae844b64a8ee, ASSIGN to state=OFFLINE, location=null, table=test, region=8a818d7f8283ee7f2b5fae844b64a8ee to restore RIT 2024-11-20T03:00:57,021 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f928be6f5d7a3631384ebd87c8f80996, ASSIGN to state=OFFLINE, location=null, table=test, region=f928be6f5d7a3631384ebd87c8f80996 to restore RIT 2024-11-20T03:00:57,021 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732071687021 2024-11-20T03:00:57,021 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-20T03:00:57,021 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf73c409fe50a78c430f016d1431abfb, server=1c82b83631a2,42975,1732071649704:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1c82b83631a2,43333,1732071649765, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T03:00:57,022 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-20T03:00:57,023 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-20T03:00:57,023 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=75, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure f928be6f5d7a3631384ebd87c8f80996, server=1c82b83631a2,36133,1732071649570 2024-11-20T03:00:57,023 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T03:00:57,024 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=75, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure f928be6f5d7a3631384ebd87c8f80996, server=1c82b83631a2,36133,1732071649570 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T03:00:57,032 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f63a07d19bc4399768e352bd9fcf3523, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,032 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=edd7cafc43290cb2489e8ba273907acd, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,032 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ccd7aea6ec14848f8c6730c88ce1c780, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,032 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=63b717291dd2cfc1f752c75b18ccc26b, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,033 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2fe87cdaf67461ff5c508d851e418c66, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,033 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fc28de84807b58cb0d6ceaa73127debb, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,033 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3c5e70743a336dce38e44ee0f8059378, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,033 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2816319e15be2a1534d61304db509629, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,034 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f928be6f5d7a3631384ebd87c8f80996, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,034 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad7c736680f4a75df049e62d21f77c48, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,034 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=56a64fab34378d9775e0bdf29862f7e6, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,034 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=535a88ef9d35d9795c43261c431087a8, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,035 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=38fa6330e381ad6ba7391bf9050c289e, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,035 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fbf7ef74de7c98db6b37604b08e67282, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,035 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4327d986b4dd716142908b96606b0ad7, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,035 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ef70776250de5f4e8ad7bd24027abe63, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,035 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=30cc99ff17ee8d21b793ce798b4998ba, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,036 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a641a5878ac82d20c96fa3dff0720791, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,036 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=550bbfe722915166ce74fec36828c508, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,036 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a818d7f8283ee7f2b5fae844b64a8ee, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,036 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=49f9e405ee146272624d3d205a4ebf9e, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,036 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=45e8fc1ea8df34f038aeb00318494097, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,037 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a06021a0c54a875234d29e87253d48a, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,037 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e8e6d28d8e1806fa7d958327b87df211, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,037 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=295cabe34d2d3dbccccb06cf3aa5bbf6, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,037 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bf73c409fe50a78c430f016d1431abfb, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,037 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-20T03:00:57,037 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732071717037 2024-11-20T03:00:57,037 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732071777037 2024-11-20T03:00:57,037 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 16 msec 2024-11-20T03:00:57,037 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-20T03:00:57,038 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T03:00:57,038 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T03:00:57,038 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T03:00:57,038 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-20T03:00:57,038 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-20T03:00:57,038 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-20T03:00:57,039 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T03:00:57,039 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-20T03:00:57,044 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-20T03:00:57,044 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T03:00:57,044 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T03:00:57,044 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T03:00:57,044 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T03:00:57,044 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T03:00:57,044 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T03:00:57,044 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T03:00:57,044 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T03:00:57,044 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T03:00:57,044 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T03:00:57,044 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T03:00:57,044 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T03:00:57,044 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T03:00:57,045 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T03:00:57,045 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T03:00:57,045 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T03:00:57,045 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T03:00:57,045 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T03:00:57,045 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T03:00:57,046 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-20T03:00:57,049 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf73c409fe50a78c430f016d1431abfb, server=1c82b83631a2,42975,1732071649704:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1c82b83631a2,43333,1732071649765, splitWal=true, meta=true 2024-11-20T03:00:57,049 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 1c82b83631a2%2C43333%2C1732071649765.meta.1732071651840.meta, worker=1c82b83631a2,36133,1732071649570 2024-11-20T03:00:57,049 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1c82b83631a2%2C43333%2C1732071649765.meta.1732071651840.meta, worker=1c82b83631a2,36133,1732071649570 2024-11-20T03:00:57,049 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T03:00:57,049 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1c82b83631a2,42975,1732071649704 2024-11-20T03:00:57,049 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 1c82b83631a2%2C43333%2C1732071649765.1732071651367, worker=1c82b83631a2,42975,1732071649704 2024-11-20T03:00:57,049 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1c82b83631a2%2C43333%2C1732071649765.1732071651367, worker=1c82b83631a2,42975,1732071649704 2024-11-20T03:00:57,049 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=38fa6330e381ad6ba7391bf9050c289e, ASSIGN 2024-11-20T03:00:57,049 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf73c409fe50a78c430f016d1431abfb, ASSIGN 2024-11-20T03:00:57,049 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ccd7aea6ec14848f8c6730c88ce1c780, ASSIGN 2024-11-20T03:00:57,049 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8a06021a0c54a875234d29e87253d48a, ASSIGN 2024-11-20T03:00:57,049 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4327d986b4dd716142908b96606b0ad7, ASSIGN 2024-11-20T03:00:57,049 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fc28de84807b58cb0d6ceaa73127debb, ASSIGN 2024-11-20T03:00:57,049 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8a818d7f8283ee7f2b5fae844b64a8ee, ASSIGN 2024-11-20T03:00:57,049 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f928be6f5d7a3631384ebd87c8f80996, ASSIGN 2024-11-20T03:00:57,050 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=65, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf73c409fe50a78c430f016d1431abfb, server=1c82b83631a2,42975,1732071649704:java.lang.RuntimeException: inject code bug; OpenRegionProcedure bf73c409fe50a78c430f016d1431abfb, server=1c82b83631a2,42975,1732071649704 2024-11-20T03:00:57,050 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure fc28de84807b58cb0d6ceaa73127debb, server=1c82b83631a2,42975,1732071649704 2024-11-20T03:00:57,050 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure ccd7aea6ec14848f8c6730c88ce1c780, server=1c82b83631a2,36133,1732071649570 2024-11-20T03:00:57,050 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure f928be6f5d7a3631384ebd87c8f80996, server=1c82b83631a2,36133,1732071649570 2024-11-20T03:00:57,050 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf73c409fe50a78c430f016d1431abfb, server=1c82b83631a2,42975,1732071649704:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1c82b83631a2,43333,1732071649765, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:57,050 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 1c82b83631a2%2C43333%2C1732071649765.meta.1732071651840.meta, worker=1c82b83631a2,36133,1732071649570 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:57,050 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1c82b83631a2%2C43333%2C1732071649765.meta.1732071651840.meta, worker=1c82b83631a2,36133,1732071649570 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:57,050 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:57,050 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1c82b83631a2,42975,1732071649704 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:57,050 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 1c82b83631a2%2C43333%2C1732071649765.1732071651367, worker=1c82b83631a2,42975,1732071649704 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:57,050 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1c82b83631a2%2C43333%2C1732071649765.1732071651367, worker=1c82b83631a2,42975,1732071649704 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:57,050 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=38fa6330e381ad6ba7391bf9050c289e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:57,050 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=38fa6330e381ad6ba7391bf9050c289e, ASSIGN 2024-11-20T03:00:57,050 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf73c409fe50a78c430f016d1431abfb, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:57,050 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf73c409fe50a78c430f016d1431abfb, ASSIGN 2024-11-20T03:00:57,050 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ccd7aea6ec14848f8c6730c88ce1c780, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:57,050 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ccd7aea6ec14848f8c6730c88ce1c780, ASSIGN 2024-11-20T03:00:57,050 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8a06021a0c54a875234d29e87253d48a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:57,050 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8a06021a0c54a875234d29e87253d48a, ASSIGN 2024-11-20T03:00:57,051 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4327d986b4dd716142908b96606b0ad7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:57,051 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4327d986b4dd716142908b96606b0ad7, ASSIGN 2024-11-20T03:00:57,051 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fc28de84807b58cb0d6ceaa73127debb, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:57,051 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fc28de84807b58cb0d6ceaa73127debb, ASSIGN 2024-11-20T03:00:57,051 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8a818d7f8283ee7f2b5fae844b64a8ee, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:57,051 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8a818d7f8283ee7f2b5fae844b64a8ee, ASSIGN 2024-11-20T03:00:57,051 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f928be6f5d7a3631384ebd87c8f80996, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:57,051 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f928be6f5d7a3631384ebd87c8f80996, ASSIGN 2024-11-20T03:00:57,051 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=65, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf73c409fe50a78c430f016d1431abfb, server=1c82b83631a2,42975,1732071649704:java.lang.RuntimeException: inject code bug; OpenRegionProcedure bf73c409fe50a78c430f016d1431abfb, server=1c82b83631a2,42975,1732071649704 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:57,051 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure fc28de84807b58cb0d6ceaa73127debb, server=1c82b83631a2,42975,1732071649704 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:57,051 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure ccd7aea6ec14848f8c6730c88ce1c780, server=1c82b83631a2,36133,1732071649570 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:57,051 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure f928be6f5d7a3631384ebd87c8f80996, server=1c82b83631a2,36133,1732071649570 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:57,051 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 11 msec 2024-11-20T03:00:57,052 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=61 2024-11-20T03:00:57,052 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=1c82b83631a2,42975,1732071649704, table=hbase:meta, region=1588230740 2024-11-20T03:00:57,052 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1c82b83631a2,42975,1732071649704, state=OPEN 2024-11-20T03:00:57,122 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39617 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T03:00:57,122 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39617 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T03:00:57,241 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42975-0x10157797d910002, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T03:00:57,241 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39617-0x10157797d910000, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T03:00:57,242 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T03:00:57,242 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T03:00:57,242 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36133-0x10157797d910001, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T03:00:57,242 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T03:00:57,243 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=38fa6330e381ad6ba7391bf9050c289e, ASSIGN to state=OFFLINE, location=null, table=test, region=38fa6330e381ad6ba7391bf9050c289e to restore RIT 2024-11-20T03:00:57,244 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf73c409fe50a78c430f016d1431abfb, ASSIGN to state=OFFLINE, location=null, table=test, region=bf73c409fe50a78c430f016d1431abfb to restore RIT 2024-11-20T03:00:57,244 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ccd7aea6ec14848f8c6730c88ce1c780, ASSIGN to state=OFFLINE, location=null, table=test, region=ccd7aea6ec14848f8c6730c88ce1c780 to restore RIT 2024-11-20T03:00:57,244 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8a06021a0c54a875234d29e87253d48a, ASSIGN to state=OFFLINE, location=null, table=test, region=8a06021a0c54a875234d29e87253d48a to restore RIT 2024-11-20T03:00:57,244 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4327d986b4dd716142908b96606b0ad7, ASSIGN to state=OFFLINE, location=null, table=test, region=4327d986b4dd716142908b96606b0ad7 to restore RIT 2024-11-20T03:00:57,244 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fc28de84807b58cb0d6ceaa73127debb, ASSIGN to state=OFFLINE, location=null, table=test, region=fc28de84807b58cb0d6ceaa73127debb to restore RIT 2024-11-20T03:00:57,244 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8a818d7f8283ee7f2b5fae844b64a8ee, ASSIGN to state=OFFLINE, location=null, table=test, region=8a818d7f8283ee7f2b5fae844b64a8ee to restore RIT 2024-11-20T03:00:57,244 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f928be6f5d7a3631384ebd87c8f80996, ASSIGN to state=OFFLINE, location=null, table=test, region=f928be6f5d7a3631384ebd87c8f80996 to restore RIT 2024-11-20T03:00:57,245 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732071687245 2024-11-20T03:00:57,245 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-20T03:00:57,245 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf73c409fe50a78c430f016d1431abfb, server=1c82b83631a2,42975,1732071649704:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1c82b83631a2,43333,1732071649765, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T03:00:57,247 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-20T03:00:57,248 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-20T03:00:57,248 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=74, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure ccd7aea6ec14848f8c6730c88ce1c780, server=1c82b83631a2,36133,1732071649570 2024-11-20T03:00:57,248 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T03:00:57,249 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=74, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure ccd7aea6ec14848f8c6730c88ce1c780, server=1c82b83631a2,36133,1732071649570 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T03:00:57,253 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f63a07d19bc4399768e352bd9fcf3523, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,254 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=edd7cafc43290cb2489e8ba273907acd, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,254 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ccd7aea6ec14848f8c6730c88ce1c780, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,254 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=63b717291dd2cfc1f752c75b18ccc26b, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,254 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2fe87cdaf67461ff5c508d851e418c66, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,254 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fc28de84807b58cb0d6ceaa73127debb, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,255 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3c5e70743a336dce38e44ee0f8059378, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,255 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2816319e15be2a1534d61304db509629, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,255 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f928be6f5d7a3631384ebd87c8f80996, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,255 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad7c736680f4a75df049e62d21f77c48, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,255 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=56a64fab34378d9775e0bdf29862f7e6, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,255 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=535a88ef9d35d9795c43261c431087a8, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,255 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=38fa6330e381ad6ba7391bf9050c289e, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,256 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fbf7ef74de7c98db6b37604b08e67282, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,256 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4327d986b4dd716142908b96606b0ad7, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,256 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ef70776250de5f4e8ad7bd24027abe63, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,256 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=30cc99ff17ee8d21b793ce798b4998ba, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,256 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a641a5878ac82d20c96fa3dff0720791, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,256 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=550bbfe722915166ce74fec36828c508, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,256 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a818d7f8283ee7f2b5fae844b64a8ee, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,257 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=49f9e405ee146272624d3d205a4ebf9e, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,257 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=45e8fc1ea8df34f038aeb00318494097, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,257 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a06021a0c54a875234d29e87253d48a, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,257 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e8e6d28d8e1806fa7d958327b87df211, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,257 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=295cabe34d2d3dbccccb06cf3aa5bbf6, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,257 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bf73c409fe50a78c430f016d1431abfb, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,257 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-20T03:00:57,257 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732071717257 2024-11-20T03:00:57,258 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732071777257 2024-11-20T03:00:57,258 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 12 msec 2024-11-20T03:00:57,258 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-20T03:00:57,258 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T03:00:57,258 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T03:00:57,258 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T03:00:57,258 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-20T03:00:57,258 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-20T03:00:57,258 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-20T03:00:57,259 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T03:00:57,259 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-20T03:00:57,262 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-20T03:00:57,262 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T03:00:57,262 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T03:00:57,262 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T03:00:57,262 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T03:00:57,262 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T03:00:57,262 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T03:00:57,262 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T03:00:57,262 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T03:00:57,262 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T03:00:57,262 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T03:00:57,262 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T03:00:57,262 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T03:00:57,262 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T03:00:57,262 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T03:00:57,262 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T03:00:57,262 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T03:00:57,262 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T03:00:57,263 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T03:00:57,263 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-20T03:00:57,265 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf73c409fe50a78c430f016d1431abfb, server=1c82b83631a2,42975,1732071649704:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1c82b83631a2,43333,1732071649765, splitWal=true, meta=true 2024-11-20T03:00:57,265 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 1c82b83631a2%2C43333%2C1732071649765.meta.1732071651840.meta, worker=1c82b83631a2,36133,1732071649570 2024-11-20T03:00:57,265 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1c82b83631a2%2C43333%2C1732071649765.meta.1732071651840.meta, worker=1c82b83631a2,36133,1732071649570 2024-11-20T03:00:57,265 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T03:00:57,265 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1c82b83631a2,42975,1732071649704 2024-11-20T03:00:57,265 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 1c82b83631a2%2C43333%2C1732071649765.1732071651367, worker=1c82b83631a2,42975,1732071649704 2024-11-20T03:00:57,265 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1c82b83631a2%2C43333%2C1732071649765.1732071651367, worker=1c82b83631a2,42975,1732071649704 2024-11-20T03:00:57,265 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=38fa6330e381ad6ba7391bf9050c289e, ASSIGN 2024-11-20T03:00:57,265 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf73c409fe50a78c430f016d1431abfb, ASSIGN 2024-11-20T03:00:57,265 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ccd7aea6ec14848f8c6730c88ce1c780, ASSIGN 2024-11-20T03:00:57,265 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8a06021a0c54a875234d29e87253d48a, ASSIGN 2024-11-20T03:00:57,265 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4327d986b4dd716142908b96606b0ad7, ASSIGN 2024-11-20T03:00:57,265 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fc28de84807b58cb0d6ceaa73127debb, ASSIGN 2024-11-20T03:00:57,265 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8a818d7f8283ee7f2b5fae844b64a8ee, ASSIGN 2024-11-20T03:00:57,265 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f928be6f5d7a3631384ebd87c8f80996, ASSIGN 2024-11-20T03:00:57,265 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=65, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf73c409fe50a78c430f016d1431abfb, server=1c82b83631a2,42975,1732071649704:java.lang.RuntimeException: inject code bug; OpenRegionProcedure bf73c409fe50a78c430f016d1431abfb, server=1c82b83631a2,42975,1732071649704 2024-11-20T03:00:57,265 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure fc28de84807b58cb0d6ceaa73127debb, server=1c82b83631a2,42975,1732071649704 2024-11-20T03:00:57,265 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure ccd7aea6ec14848f8c6730c88ce1c780, server=1c82b83631a2,36133,1732071649570 2024-11-20T03:00:57,265 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf73c409fe50a78c430f016d1431abfb, server=1c82b83631a2,42975,1732071649704:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1c82b83631a2,43333,1732071649765, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:57,265 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 1c82b83631a2%2C43333%2C1732071649765.meta.1732071651840.meta, worker=1c82b83631a2,36133,1732071649570 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:57,266 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1c82b83631a2%2C43333%2C1732071649765.meta.1732071651840.meta, worker=1c82b83631a2,36133,1732071649570 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:57,266 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:57,266 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1c82b83631a2,42975,1732071649704 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:57,266 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 1c82b83631a2%2C43333%2C1732071649765.1732071651367, worker=1c82b83631a2,42975,1732071649704 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:57,266 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1c82b83631a2%2C43333%2C1732071649765.1732071651367, worker=1c82b83631a2,42975,1732071649704 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:57,266 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=38fa6330e381ad6ba7391bf9050c289e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:57,266 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=38fa6330e381ad6ba7391bf9050c289e, ASSIGN 2024-11-20T03:00:57,266 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf73c409fe50a78c430f016d1431abfb, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:57,266 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf73c409fe50a78c430f016d1431abfb, ASSIGN 2024-11-20T03:00:57,266 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ccd7aea6ec14848f8c6730c88ce1c780, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:57,266 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ccd7aea6ec14848f8c6730c88ce1c780, ASSIGN 2024-11-20T03:00:57,266 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8a06021a0c54a875234d29e87253d48a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:57,266 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8a06021a0c54a875234d29e87253d48a, ASSIGN 2024-11-20T03:00:57,266 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4327d986b4dd716142908b96606b0ad7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:57,266 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4327d986b4dd716142908b96606b0ad7, ASSIGN 2024-11-20T03:00:57,266 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fc28de84807b58cb0d6ceaa73127debb, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:57,266 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fc28de84807b58cb0d6ceaa73127debb, ASSIGN 2024-11-20T03:00:57,266 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8a818d7f8283ee7f2b5fae844b64a8ee, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:57,266 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8a818d7f8283ee7f2b5fae844b64a8ee, ASSIGN 2024-11-20T03:00:57,266 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f928be6f5d7a3631384ebd87c8f80996, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:57,266 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f928be6f5d7a3631384ebd87c8f80996, ASSIGN 2024-11-20T03:00:57,266 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=65, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf73c409fe50a78c430f016d1431abfb, server=1c82b83631a2,42975,1732071649704:java.lang.RuntimeException: inject code bug; OpenRegionProcedure bf73c409fe50a78c430f016d1431abfb, server=1c82b83631a2,42975,1732071649704 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:57,266 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure fc28de84807b58cb0d6ceaa73127debb, server=1c82b83631a2,42975,1732071649704 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:57,266 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure ccd7aea6ec14848f8c6730c88ce1c780, server=1c82b83631a2,36133,1732071649570 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:57,266 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-20T03:00:57,267 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=61 2024-11-20T03:00:57,267 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=1c82b83631a2,42975,1732071649704, table=hbase:meta, region=1588230740 2024-11-20T03:00:57,268 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1c82b83631a2,42975,1732071649704, state=OPEN 2024-11-20T03:00:57,281 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36133-0x10157797d910001, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T03:00:57,281 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39617-0x10157797d910000, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T03:00:57,281 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42975-0x10157797d910002, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T03:00:57,282 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T03:00:57,282 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T03:00:57,282 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T03:00:57,282 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=38fa6330e381ad6ba7391bf9050c289e, ASSIGN to state=OFFLINE, location=null, table=test, region=38fa6330e381ad6ba7391bf9050c289e to restore RIT 2024-11-20T03:00:57,282 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf73c409fe50a78c430f016d1431abfb, ASSIGN to state=OFFLINE, location=null, table=test, region=bf73c409fe50a78c430f016d1431abfb to restore RIT 2024-11-20T03:00:57,282 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ccd7aea6ec14848f8c6730c88ce1c780, ASSIGN to state=OFFLINE, location=null, table=test, region=ccd7aea6ec14848f8c6730c88ce1c780 to restore RIT 2024-11-20T03:00:57,282 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8a06021a0c54a875234d29e87253d48a, ASSIGN to state=OFFLINE, location=null, table=test, region=8a06021a0c54a875234d29e87253d48a to restore RIT 2024-11-20T03:00:57,282 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4327d986b4dd716142908b96606b0ad7, ASSIGN to state=OFFLINE, location=null, table=test, region=4327d986b4dd716142908b96606b0ad7 to restore RIT 2024-11-20T03:00:57,282 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fc28de84807b58cb0d6ceaa73127debb, ASSIGN to state=OFFLINE, location=null, table=test, region=fc28de84807b58cb0d6ceaa73127debb to restore RIT 2024-11-20T03:00:57,282 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8a818d7f8283ee7f2b5fae844b64a8ee, ASSIGN to state=OFFLINE, location=null, table=test, region=8a818d7f8283ee7f2b5fae844b64a8ee to restore RIT 2024-11-20T03:00:57,283 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f928be6f5d7a3631384ebd87c8f80996, ASSIGN to state=OFFLINE, location=null, table=test, region=f928be6f5d7a3631384ebd87c8f80996 to restore RIT 2024-11-20T03:00:57,283 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732071687283 2024-11-20T03:00:57,283 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-20T03:00:57,283 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf73c409fe50a78c430f016d1431abfb, server=1c82b83631a2,42975,1732071649704:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1c82b83631a2,43333,1732071649765, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T03:00:57,284 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-20T03:00:57,285 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-20T03:00:57,285 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=73, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure fc28de84807b58cb0d6ceaa73127debb, server=1c82b83631a2,42975,1732071649704 2024-11-20T03:00:57,285 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T03:00:57,286 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=73, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure fc28de84807b58cb0d6ceaa73127debb, server=1c82b83631a2,42975,1732071649704 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T03:00:57,288 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f63a07d19bc4399768e352bd9fcf3523, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,288 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=edd7cafc43290cb2489e8ba273907acd, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,288 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ccd7aea6ec14848f8c6730c88ce1c780, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,289 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=63b717291dd2cfc1f752c75b18ccc26b, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,289 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2fe87cdaf67461ff5c508d851e418c66, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,289 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fc28de84807b58cb0d6ceaa73127debb, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,289 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3c5e70743a336dce38e44ee0f8059378, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,289 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2816319e15be2a1534d61304db509629, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,289 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f928be6f5d7a3631384ebd87c8f80996, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,289 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad7c736680f4a75df049e62d21f77c48, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,289 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=56a64fab34378d9775e0bdf29862f7e6, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,289 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=535a88ef9d35d9795c43261c431087a8, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,290 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=38fa6330e381ad6ba7391bf9050c289e, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,290 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fbf7ef74de7c98db6b37604b08e67282, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,290 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4327d986b4dd716142908b96606b0ad7, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,290 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ef70776250de5f4e8ad7bd24027abe63, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,290 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=30cc99ff17ee8d21b793ce798b4998ba, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,290 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a641a5878ac82d20c96fa3dff0720791, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,290 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=550bbfe722915166ce74fec36828c508, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,290 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a818d7f8283ee7f2b5fae844b64a8ee, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,290 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=49f9e405ee146272624d3d205a4ebf9e, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,291 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=45e8fc1ea8df34f038aeb00318494097, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,291 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a06021a0c54a875234d29e87253d48a, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,291 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e8e6d28d8e1806fa7d958327b87df211, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,291 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=295cabe34d2d3dbccccb06cf3aa5bbf6, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,291 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bf73c409fe50a78c430f016d1431abfb, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,291 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-20T03:00:57,291 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732071717291 2024-11-20T03:00:57,291 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732071777291 2024-11-20T03:00:57,291 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-11-20T03:00:57,291 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-20T03:00:57,292 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T03:00:57,292 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T03:00:57,292 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T03:00:57,292 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-20T03:00:57,292 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-20T03:00:57,292 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-20T03:00:57,292 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T03:00:57,293 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-20T03:00:57,295 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-20T03:00:57,295 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T03:00:57,295 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T03:00:57,295 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T03:00:57,295 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T03:00:57,295 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T03:00:57,295 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T03:00:57,295 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T03:00:57,295 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T03:00:57,295 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T03:00:57,295 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T03:00:57,295 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T03:00:57,295 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T03:00:57,295 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T03:00:57,295 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T03:00:57,295 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T03:00:57,295 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T03:00:57,296 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T03:00:57,296 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-20T03:00:57,298 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf73c409fe50a78c430f016d1431abfb, server=1c82b83631a2,42975,1732071649704:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1c82b83631a2,43333,1732071649765, splitWal=true, meta=true 2024-11-20T03:00:57,298 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 1c82b83631a2%2C43333%2C1732071649765.meta.1732071651840.meta, worker=1c82b83631a2,36133,1732071649570 2024-11-20T03:00:57,298 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1c82b83631a2%2C43333%2C1732071649765.meta.1732071651840.meta, worker=1c82b83631a2,36133,1732071649570 2024-11-20T03:00:57,298 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T03:00:57,298 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1c82b83631a2,42975,1732071649704 2024-11-20T03:00:57,298 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 1c82b83631a2%2C43333%2C1732071649765.1732071651367, worker=1c82b83631a2,42975,1732071649704 2024-11-20T03:00:57,298 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1c82b83631a2%2C43333%2C1732071649765.1732071651367, worker=1c82b83631a2,42975,1732071649704 2024-11-20T03:00:57,298 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=38fa6330e381ad6ba7391bf9050c289e, ASSIGN 2024-11-20T03:00:57,298 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf73c409fe50a78c430f016d1431abfb, ASSIGN 2024-11-20T03:00:57,298 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ccd7aea6ec14848f8c6730c88ce1c780, ASSIGN 2024-11-20T03:00:57,298 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8a06021a0c54a875234d29e87253d48a, ASSIGN 2024-11-20T03:00:57,298 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4327d986b4dd716142908b96606b0ad7, ASSIGN 2024-11-20T03:00:57,298 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fc28de84807b58cb0d6ceaa73127debb, ASSIGN 2024-11-20T03:00:57,298 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8a818d7f8283ee7f2b5fae844b64a8ee, ASSIGN 2024-11-20T03:00:57,298 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f928be6f5d7a3631384ebd87c8f80996, ASSIGN 2024-11-20T03:00:57,298 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=65, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf73c409fe50a78c430f016d1431abfb, server=1c82b83631a2,42975,1732071649704:java.lang.RuntimeException: inject code bug; OpenRegionProcedure bf73c409fe50a78c430f016d1431abfb, server=1c82b83631a2,42975,1732071649704 2024-11-20T03:00:57,298 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure fc28de84807b58cb0d6ceaa73127debb, server=1c82b83631a2,42975,1732071649704 2024-11-20T03:00:57,298 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf73c409fe50a78c430f016d1431abfb, server=1c82b83631a2,42975,1732071649704:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1c82b83631a2,43333,1732071649765, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:57,298 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 1c82b83631a2%2C43333%2C1732071649765.meta.1732071651840.meta, worker=1c82b83631a2,36133,1732071649570 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:57,298 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1c82b83631a2%2C43333%2C1732071649765.meta.1732071651840.meta, worker=1c82b83631a2,36133,1732071649570 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:57,298 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:57,298 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1c82b83631a2,42975,1732071649704 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:57,298 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 1c82b83631a2%2C43333%2C1732071649765.1732071651367, worker=1c82b83631a2,42975,1732071649704 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:57,298 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1c82b83631a2%2C43333%2C1732071649765.1732071651367, worker=1c82b83631a2,42975,1732071649704 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:57,298 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=38fa6330e381ad6ba7391bf9050c289e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:57,299 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=38fa6330e381ad6ba7391bf9050c289e, ASSIGN 2024-11-20T03:00:57,299 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf73c409fe50a78c430f016d1431abfb, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:57,299 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf73c409fe50a78c430f016d1431abfb, ASSIGN 2024-11-20T03:00:57,299 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ccd7aea6ec14848f8c6730c88ce1c780, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:57,299 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ccd7aea6ec14848f8c6730c88ce1c780, ASSIGN 2024-11-20T03:00:57,299 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8a06021a0c54a875234d29e87253d48a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:57,299 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8a06021a0c54a875234d29e87253d48a, ASSIGN 2024-11-20T03:00:57,299 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4327d986b4dd716142908b96606b0ad7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:57,299 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4327d986b4dd716142908b96606b0ad7, ASSIGN 2024-11-20T03:00:57,299 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fc28de84807b58cb0d6ceaa73127debb, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:57,299 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fc28de84807b58cb0d6ceaa73127debb, ASSIGN 2024-11-20T03:00:57,299 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8a818d7f8283ee7f2b5fae844b64a8ee, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:57,299 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8a818d7f8283ee7f2b5fae844b64a8ee, ASSIGN 2024-11-20T03:00:57,299 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f928be6f5d7a3631384ebd87c8f80996, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:57,299 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f928be6f5d7a3631384ebd87c8f80996, ASSIGN 2024-11-20T03:00:57,299 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=65, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf73c409fe50a78c430f016d1431abfb, server=1c82b83631a2,42975,1732071649704:java.lang.RuntimeException: inject code bug; OpenRegionProcedure bf73c409fe50a78c430f016d1431abfb, server=1c82b83631a2,42975,1732071649704 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:57,299 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure fc28de84807b58cb0d6ceaa73127debb, server=1c82b83631a2,42975,1732071649704 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:57,299 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-20T03:00:57,300 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=61 2024-11-20T03:00:57,300 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=1c82b83631a2,42975,1732071649704, table=hbase:meta, region=1588230740 2024-11-20T03:00:57,300 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1c82b83631a2,42975,1732071649704, state=OPEN 2024-11-20T03:00:57,306 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42975-0x10157797d910002, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T03:00:57,306 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36133-0x10157797d910001, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T03:00:57,306 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39617-0x10157797d910000, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T03:00:57,307 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T03:00:57,307 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T03:00:57,307 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T03:00:57,307 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=38fa6330e381ad6ba7391bf9050c289e, ASSIGN to state=OFFLINE, location=null, table=test, region=38fa6330e381ad6ba7391bf9050c289e to restore RIT 2024-11-20T03:00:57,307 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf73c409fe50a78c430f016d1431abfb, ASSIGN to state=OFFLINE, location=null, table=test, region=bf73c409fe50a78c430f016d1431abfb to restore RIT 2024-11-20T03:00:57,307 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ccd7aea6ec14848f8c6730c88ce1c780, ASSIGN to state=OFFLINE, location=null, table=test, region=ccd7aea6ec14848f8c6730c88ce1c780 to restore RIT 2024-11-20T03:00:57,307 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8a06021a0c54a875234d29e87253d48a, ASSIGN to state=OFFLINE, location=null, table=test, region=8a06021a0c54a875234d29e87253d48a to restore RIT 2024-11-20T03:00:57,307 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4327d986b4dd716142908b96606b0ad7, ASSIGN to state=OFFLINE, location=null, table=test, region=4327d986b4dd716142908b96606b0ad7 to restore RIT 2024-11-20T03:00:57,307 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fc28de84807b58cb0d6ceaa73127debb, ASSIGN to state=OFFLINE, location=null, table=test, region=fc28de84807b58cb0d6ceaa73127debb to restore RIT 2024-11-20T03:00:57,307 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8a818d7f8283ee7f2b5fae844b64a8ee, ASSIGN to state=OFFLINE, location=null, table=test, region=8a818d7f8283ee7f2b5fae844b64a8ee to restore RIT 2024-11-20T03:00:57,307 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f928be6f5d7a3631384ebd87c8f80996, ASSIGN to state=OFFLINE, location=null, table=test, region=f928be6f5d7a3631384ebd87c8f80996 to restore RIT 2024-11-20T03:00:57,308 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732071687308 2024-11-20T03:00:57,308 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-20T03:00:57,308 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf73c409fe50a78c430f016d1431abfb, server=1c82b83631a2,42975,1732071649704:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1c82b83631a2,43333,1732071649765, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T03:00:57,309 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-20T03:00:57,310 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-20T03:00:57,310 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=72, ppid=65, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf73c409fe50a78c430f016d1431abfb, server=1c82b83631a2,42975,1732071649704:java.lang.RuntimeException: inject code bug; OpenRegionProcedure bf73c409fe50a78c430f016d1431abfb, server=1c82b83631a2,42975,1732071649704 2024-11-20T03:00:57,310 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T03:00:57,310 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=72, ppid=65, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf73c409fe50a78c430f016d1431abfb, server=1c82b83631a2,42975,1732071649704:java.lang.RuntimeException: inject code bug; OpenRegionProcedure bf73c409fe50a78c430f016d1431abfb, server=1c82b83631a2,42975,1732071649704 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T03:00:57,313 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f63a07d19bc4399768e352bd9fcf3523, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,313 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=edd7cafc43290cb2489e8ba273907acd, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,313 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ccd7aea6ec14848f8c6730c88ce1c780, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,313 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=63b717291dd2cfc1f752c75b18ccc26b, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,314 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2fe87cdaf67461ff5c508d851e418c66, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,314 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fc28de84807b58cb0d6ceaa73127debb, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,314 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3c5e70743a336dce38e44ee0f8059378, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,314 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2816319e15be2a1534d61304db509629, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,314 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f928be6f5d7a3631384ebd87c8f80996, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,315 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad7c736680f4a75df049e62d21f77c48, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,315 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=56a64fab34378d9775e0bdf29862f7e6, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,315 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=535a88ef9d35d9795c43261c431087a8, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,315 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=38fa6330e381ad6ba7391bf9050c289e, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,315 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fbf7ef74de7c98db6b37604b08e67282, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,315 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4327d986b4dd716142908b96606b0ad7, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,316 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ef70776250de5f4e8ad7bd24027abe63, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,316 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=30cc99ff17ee8d21b793ce798b4998ba, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,316 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a641a5878ac82d20c96fa3dff0720791, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,316 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=550bbfe722915166ce74fec36828c508, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,316 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a818d7f8283ee7f2b5fae844b64a8ee, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,316 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=49f9e405ee146272624d3d205a4ebf9e, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,317 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=45e8fc1ea8df34f038aeb00318494097, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,317 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a06021a0c54a875234d29e87253d48a, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,317 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e8e6d28d8e1806fa7d958327b87df211, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,317 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=295cabe34d2d3dbccccb06cf3aa5bbf6, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,317 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bf73c409fe50a78c430f016d1431abfb, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,317 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-20T03:00:57,317 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732071717317 2024-11-20T03:00:57,317 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732071777317 2024-11-20T03:00:57,317 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-11-20T03:00:57,318 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-20T03:00:57,318 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T03:00:57,318 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T03:00:57,318 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T03:00:57,318 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-20T03:00:57,318 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-20T03:00:57,318 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-20T03:00:57,318 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T03:00:57,319 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-20T03:00:57,321 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-20T03:00:57,322 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T03:00:57,322 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T03:00:57,322 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T03:00:57,322 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T03:00:57,322 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T03:00:57,322 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T03:00:57,322 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T03:00:57,322 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T03:00:57,322 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T03:00:57,322 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T03:00:57,322 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T03:00:57,322 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T03:00:57,322 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T03:00:57,322 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T03:00:57,322 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T03:00:57,323 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T03:00:57,323 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-20T03:00:57,326 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf73c409fe50a78c430f016d1431abfb, server=1c82b83631a2,42975,1732071649704:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1c82b83631a2,43333,1732071649765, splitWal=true, meta=true 2024-11-20T03:00:57,326 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 1c82b83631a2%2C43333%2C1732071649765.meta.1732071651840.meta, worker=1c82b83631a2,36133,1732071649570 2024-11-20T03:00:57,326 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1c82b83631a2%2C43333%2C1732071649765.meta.1732071651840.meta, worker=1c82b83631a2,36133,1732071649570 2024-11-20T03:00:57,326 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T03:00:57,326 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1c82b83631a2,42975,1732071649704 2024-11-20T03:00:57,326 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 1c82b83631a2%2C43333%2C1732071649765.1732071651367, worker=1c82b83631a2,42975,1732071649704 2024-11-20T03:00:57,326 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1c82b83631a2%2C43333%2C1732071649765.1732071651367, worker=1c82b83631a2,42975,1732071649704 2024-11-20T03:00:57,326 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=38fa6330e381ad6ba7391bf9050c289e, ASSIGN 2024-11-20T03:00:57,326 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf73c409fe50a78c430f016d1431abfb, ASSIGN 2024-11-20T03:00:57,326 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ccd7aea6ec14848f8c6730c88ce1c780, ASSIGN 2024-11-20T03:00:57,326 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8a06021a0c54a875234d29e87253d48a, ASSIGN 2024-11-20T03:00:57,326 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4327d986b4dd716142908b96606b0ad7, ASSIGN 2024-11-20T03:00:57,326 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fc28de84807b58cb0d6ceaa73127debb, ASSIGN 2024-11-20T03:00:57,326 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8a818d7f8283ee7f2b5fae844b64a8ee, ASSIGN 2024-11-20T03:00:57,327 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f928be6f5d7a3631384ebd87c8f80996, ASSIGN 2024-11-20T03:00:57,327 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=65, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf73c409fe50a78c430f016d1431abfb, server=1c82b83631a2,42975,1732071649704:java.lang.RuntimeException: inject code bug; OpenRegionProcedure bf73c409fe50a78c430f016d1431abfb, server=1c82b83631a2,42975,1732071649704 2024-11-20T03:00:57,327 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf73c409fe50a78c430f016d1431abfb, server=1c82b83631a2,42975,1732071649704:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1c82b83631a2,43333,1732071649765, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:57,327 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 1c82b83631a2%2C43333%2C1732071649765.meta.1732071651840.meta, worker=1c82b83631a2,36133,1732071649570 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:57,327 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1c82b83631a2%2C43333%2C1732071649765.meta.1732071651840.meta, worker=1c82b83631a2,36133,1732071649570 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:57,327 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:57,327 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1c82b83631a2,42975,1732071649704 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:57,327 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 1c82b83631a2%2C43333%2C1732071649765.1732071651367, worker=1c82b83631a2,42975,1732071649704 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:57,327 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1c82b83631a2%2C43333%2C1732071649765.1732071651367, worker=1c82b83631a2,42975,1732071649704 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:57,327 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=38fa6330e381ad6ba7391bf9050c289e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:57,327 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=38fa6330e381ad6ba7391bf9050c289e, ASSIGN 2024-11-20T03:00:57,327 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf73c409fe50a78c430f016d1431abfb, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:57,327 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf73c409fe50a78c430f016d1431abfb, ASSIGN 2024-11-20T03:00:57,327 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ccd7aea6ec14848f8c6730c88ce1c780, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:57,327 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ccd7aea6ec14848f8c6730c88ce1c780, ASSIGN 2024-11-20T03:00:57,327 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8a06021a0c54a875234d29e87253d48a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:57,327 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8a06021a0c54a875234d29e87253d48a, ASSIGN 2024-11-20T03:00:57,327 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4327d986b4dd716142908b96606b0ad7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:57,327 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4327d986b4dd716142908b96606b0ad7, ASSIGN 2024-11-20T03:00:57,327 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fc28de84807b58cb0d6ceaa73127debb, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:57,328 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fc28de84807b58cb0d6ceaa73127debb, ASSIGN 2024-11-20T03:00:57,328 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8a818d7f8283ee7f2b5fae844b64a8ee, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:57,328 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8a818d7f8283ee7f2b5fae844b64a8ee, ASSIGN 2024-11-20T03:00:57,328 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f928be6f5d7a3631384ebd87c8f80996, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:57,328 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f928be6f5d7a3631384ebd87c8f80996, ASSIGN 2024-11-20T03:00:57,328 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=65, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf73c409fe50a78c430f016d1431abfb, server=1c82b83631a2,42975,1732071649704:java.lang.RuntimeException: inject code bug; OpenRegionProcedure bf73c409fe50a78c430f016d1431abfb, server=1c82b83631a2,42975,1732071649704 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:57,328 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 8 msec 2024-11-20T03:00:57,329 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=61 2024-11-20T03:00:57,329 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=1c82b83631a2,42975,1732071649704, table=hbase:meta, region=1588230740 2024-11-20T03:00:57,329 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1c82b83631a2,42975,1732071649704, state=OPEN 2024-11-20T03:00:57,340 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39617-0x10157797d910000, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T03:00:57,340 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42975-0x10157797d910002, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T03:00:57,340 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36133-0x10157797d910001, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T03:00:57,340 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T03:00:57,340 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T03:00:57,340 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T03:00:57,341 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=38fa6330e381ad6ba7391bf9050c289e, ASSIGN to state=OFFLINE, location=null, table=test, region=38fa6330e381ad6ba7391bf9050c289e to restore RIT 2024-11-20T03:00:57,341 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf73c409fe50a78c430f016d1431abfb, ASSIGN to state=OFFLINE, location=null, table=test, region=bf73c409fe50a78c430f016d1431abfb to restore RIT 2024-11-20T03:00:57,341 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ccd7aea6ec14848f8c6730c88ce1c780, ASSIGN to state=OFFLINE, location=null, table=test, region=ccd7aea6ec14848f8c6730c88ce1c780 to restore RIT 2024-11-20T03:00:57,341 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8a06021a0c54a875234d29e87253d48a, ASSIGN to state=OFFLINE, location=null, table=test, region=8a06021a0c54a875234d29e87253d48a to restore RIT 2024-11-20T03:00:57,341 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4327d986b4dd716142908b96606b0ad7, ASSIGN to state=OFFLINE, location=null, table=test, region=4327d986b4dd716142908b96606b0ad7 to restore RIT 2024-11-20T03:00:57,341 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fc28de84807b58cb0d6ceaa73127debb, ASSIGN to state=OFFLINE, location=null, table=test, region=fc28de84807b58cb0d6ceaa73127debb to restore RIT 2024-11-20T03:00:57,341 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8a818d7f8283ee7f2b5fae844b64a8ee, ASSIGN to state=OFFLINE, location=null, table=test, region=8a818d7f8283ee7f2b5fae844b64a8ee to restore RIT 2024-11-20T03:00:57,341 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f928be6f5d7a3631384ebd87c8f80996, ASSIGN to state=OFFLINE, location=null, table=test, region=f928be6f5d7a3631384ebd87c8f80996 to restore RIT 2024-11-20T03:00:57,341 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732071687341 2024-11-20T03:00:57,342 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-20T03:00:57,342 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf73c409fe50a78c430f016d1431abfb, server=1c82b83631a2,42975,1732071649704:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1c82b83631a2,43333,1732071649765, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T03:00:57,343 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-20T03:00:57,346 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-20T03:00:57,346 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f928be6f5d7a3631384ebd87c8f80996, ASSIGN 2024-11-20T03:00:57,346 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T03:00:57,346 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f928be6f5d7a3631384ebd87c8f80996, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T03:00:57,348 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f63a07d19bc4399768e352bd9fcf3523, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,348 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=edd7cafc43290cb2489e8ba273907acd, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,348 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ccd7aea6ec14848f8c6730c88ce1c780, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,348 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=63b717291dd2cfc1f752c75b18ccc26b, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,348 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2fe87cdaf67461ff5c508d851e418c66, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,349 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fc28de84807b58cb0d6ceaa73127debb, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,349 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3c5e70743a336dce38e44ee0f8059378, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,349 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2816319e15be2a1534d61304db509629, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,349 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f928be6f5d7a3631384ebd87c8f80996, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,349 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad7c736680f4a75df049e62d21f77c48, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,349 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=56a64fab34378d9775e0bdf29862f7e6, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,349 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=535a88ef9d35d9795c43261c431087a8, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,349 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=38fa6330e381ad6ba7391bf9050c289e, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,350 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fbf7ef74de7c98db6b37604b08e67282, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,350 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4327d986b4dd716142908b96606b0ad7, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,350 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ef70776250de5f4e8ad7bd24027abe63, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,350 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=30cc99ff17ee8d21b793ce798b4998ba, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,350 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a641a5878ac82d20c96fa3dff0720791, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,350 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=550bbfe722915166ce74fec36828c508, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,350 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a818d7f8283ee7f2b5fae844b64a8ee, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,350 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=49f9e405ee146272624d3d205a4ebf9e, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,351 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=45e8fc1ea8df34f038aeb00318494097, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,351 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a06021a0c54a875234d29e87253d48a, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,351 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e8e6d28d8e1806fa7d958327b87df211, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,351 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=295cabe34d2d3dbccccb06cf3aa5bbf6, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,351 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bf73c409fe50a78c430f016d1431abfb, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,351 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-20T03:00:57,351 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732071717351 2024-11-20T03:00:57,351 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732071777351 2024-11-20T03:00:57,351 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-11-20T03:00:57,351 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-20T03:00:57,351 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T03:00:57,351 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T03:00:57,351 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T03:00:57,352 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-20T03:00:57,352 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-20T03:00:57,352 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-20T03:00:57,352 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T03:00:57,352 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-20T03:00:57,354 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-20T03:00:57,354 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T03:00:57,354 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T03:00:57,354 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T03:00:57,354 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T03:00:57,354 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T03:00:57,354 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T03:00:57,354 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T03:00:57,354 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T03:00:57,354 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T03:00:57,354 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T03:00:57,354 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T03:00:57,354 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T03:00:57,354 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T03:00:57,354 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T03:00:57,355 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T03:00:57,355 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-20T03:00:57,356 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf73c409fe50a78c430f016d1431abfb, server=1c82b83631a2,42975,1732071649704:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1c82b83631a2,43333,1732071649765, splitWal=true, meta=true 2024-11-20T03:00:57,356 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 1c82b83631a2%2C43333%2C1732071649765.meta.1732071651840.meta, worker=1c82b83631a2,36133,1732071649570 2024-11-20T03:00:57,356 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1c82b83631a2%2C43333%2C1732071649765.meta.1732071651840.meta, worker=1c82b83631a2,36133,1732071649570 2024-11-20T03:00:57,357 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T03:00:57,357 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1c82b83631a2,42975,1732071649704 2024-11-20T03:00:57,357 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 1c82b83631a2%2C43333%2C1732071649765.1732071651367, worker=1c82b83631a2,42975,1732071649704 2024-11-20T03:00:57,357 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1c82b83631a2%2C43333%2C1732071649765.1732071651367, worker=1c82b83631a2,42975,1732071649704 2024-11-20T03:00:57,357 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=38fa6330e381ad6ba7391bf9050c289e, ASSIGN 2024-11-20T03:00:57,357 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf73c409fe50a78c430f016d1431abfb, ASSIGN 2024-11-20T03:00:57,357 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ccd7aea6ec14848f8c6730c88ce1c780, ASSIGN 2024-11-20T03:00:57,357 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8a06021a0c54a875234d29e87253d48a, ASSIGN 2024-11-20T03:00:57,357 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4327d986b4dd716142908b96606b0ad7, ASSIGN 2024-11-20T03:00:57,357 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fc28de84807b58cb0d6ceaa73127debb, ASSIGN 2024-11-20T03:00:57,357 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8a818d7f8283ee7f2b5fae844b64a8ee, ASSIGN 2024-11-20T03:00:57,357 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f928be6f5d7a3631384ebd87c8f80996, ASSIGN 2024-11-20T03:00:57,357 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf73c409fe50a78c430f016d1431abfb, server=1c82b83631a2,42975,1732071649704:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1c82b83631a2,43333,1732071649765, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:57,357 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 1c82b83631a2%2C43333%2C1732071649765.meta.1732071651840.meta, worker=1c82b83631a2,36133,1732071649570 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:57,357 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1c82b83631a2%2C43333%2C1732071649765.meta.1732071651840.meta, worker=1c82b83631a2,36133,1732071649570 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:57,357 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:57,357 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1c82b83631a2,42975,1732071649704 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:57,357 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 1c82b83631a2%2C43333%2C1732071649765.1732071651367, worker=1c82b83631a2,42975,1732071649704 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:57,357 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1c82b83631a2%2C43333%2C1732071649765.1732071651367, worker=1c82b83631a2,42975,1732071649704 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:57,357 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=38fa6330e381ad6ba7391bf9050c289e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:57,357 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=38fa6330e381ad6ba7391bf9050c289e, ASSIGN 2024-11-20T03:00:57,357 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf73c409fe50a78c430f016d1431abfb, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:57,357 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf73c409fe50a78c430f016d1431abfb, ASSIGN 2024-11-20T03:00:57,357 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ccd7aea6ec14848f8c6730c88ce1c780, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:57,357 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ccd7aea6ec14848f8c6730c88ce1c780, ASSIGN 2024-11-20T03:00:57,357 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8a06021a0c54a875234d29e87253d48a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:57,357 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8a06021a0c54a875234d29e87253d48a, ASSIGN 2024-11-20T03:00:57,358 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4327d986b4dd716142908b96606b0ad7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:57,358 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4327d986b4dd716142908b96606b0ad7, ASSIGN 2024-11-20T03:00:57,358 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fc28de84807b58cb0d6ceaa73127debb, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:57,358 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fc28de84807b58cb0d6ceaa73127debb, ASSIGN 2024-11-20T03:00:57,358 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8a818d7f8283ee7f2b5fae844b64a8ee, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:57,358 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8a818d7f8283ee7f2b5fae844b64a8ee, ASSIGN 2024-11-20T03:00:57,358 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f928be6f5d7a3631384ebd87c8f80996, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:57,358 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f928be6f5d7a3631384ebd87c8f80996, ASSIGN 2024-11-20T03:00:57,358 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-20T03:00:57,359 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=61 2024-11-20T03:00:57,359 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=1c82b83631a2,42975,1732071649704, table=hbase:meta, region=1588230740 2024-11-20T03:00:57,359 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1c82b83631a2,42975,1732071649704, state=OPEN 2024-11-20T03:00:57,365 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36133-0x10157797d910001, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T03:00:57,365 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39617-0x10157797d910000, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T03:00:57,365 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42975-0x10157797d910002, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T03:00:57,365 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T03:00:57,365 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T03:00:57,365 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T03:00:57,365 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=38fa6330e381ad6ba7391bf9050c289e, ASSIGN to state=OFFLINE, location=null, table=test, region=38fa6330e381ad6ba7391bf9050c289e to restore RIT 2024-11-20T03:00:57,366 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf73c409fe50a78c430f016d1431abfb, ASSIGN to state=OFFLINE, location=null, table=test, region=bf73c409fe50a78c430f016d1431abfb to restore RIT 2024-11-20T03:00:57,366 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ccd7aea6ec14848f8c6730c88ce1c780, ASSIGN to state=OFFLINE, location=null, table=test, region=ccd7aea6ec14848f8c6730c88ce1c780 to restore RIT 2024-11-20T03:00:57,366 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8a06021a0c54a875234d29e87253d48a, ASSIGN to state=OFFLINE, location=null, table=test, region=8a06021a0c54a875234d29e87253d48a to restore RIT 2024-11-20T03:00:57,366 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4327d986b4dd716142908b96606b0ad7, ASSIGN to state=OFFLINE, location=null, table=test, region=4327d986b4dd716142908b96606b0ad7 to restore RIT 2024-11-20T03:00:57,366 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fc28de84807b58cb0d6ceaa73127debb, ASSIGN to state=OFFLINE, location=null, table=test, region=fc28de84807b58cb0d6ceaa73127debb to restore RIT 2024-11-20T03:00:57,366 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8a818d7f8283ee7f2b5fae844b64a8ee, ASSIGN to state=OFFLINE, location=null, table=test, region=8a818d7f8283ee7f2b5fae844b64a8ee to restore RIT 2024-11-20T03:00:57,366 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f928be6f5d7a3631384ebd87c8f80996, ASSIGN to state=OFFLINE, location=null, table=test, region=f928be6f5d7a3631384ebd87c8f80996 to restore RIT 2024-11-20T03:00:57,366 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732071687366 2024-11-20T03:00:57,366 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-20T03:00:57,366 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf73c409fe50a78c430f016d1431abfb, server=1c82b83631a2,42975,1732071649704:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1c82b83631a2,43333,1732071649765, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T03:00:57,367 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-20T03:00:57,368 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-20T03:00:57,368 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8a818d7f8283ee7f2b5fae844b64a8ee, ASSIGN 2024-11-20T03:00:57,368 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T03:00:57,369 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8a818d7f8283ee7f2b5fae844b64a8ee, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T03:00:57,371 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f63a07d19bc4399768e352bd9fcf3523, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,371 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=edd7cafc43290cb2489e8ba273907acd, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,371 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ccd7aea6ec14848f8c6730c88ce1c780, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,371 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=63b717291dd2cfc1f752c75b18ccc26b, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,372 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2fe87cdaf67461ff5c508d851e418c66, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,372 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fc28de84807b58cb0d6ceaa73127debb, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,372 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3c5e70743a336dce38e44ee0f8059378, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,372 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2816319e15be2a1534d61304db509629, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,372 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f928be6f5d7a3631384ebd87c8f80996, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,372 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad7c736680f4a75df049e62d21f77c48, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,372 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=56a64fab34378d9775e0bdf29862f7e6, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,372 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=535a88ef9d35d9795c43261c431087a8, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,372 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=38fa6330e381ad6ba7391bf9050c289e, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,373 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fbf7ef74de7c98db6b37604b08e67282, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,373 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4327d986b4dd716142908b96606b0ad7, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,373 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ef70776250de5f4e8ad7bd24027abe63, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,373 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=30cc99ff17ee8d21b793ce798b4998ba, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,373 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a641a5878ac82d20c96fa3dff0720791, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,373 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=550bbfe722915166ce74fec36828c508, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,373 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a818d7f8283ee7f2b5fae844b64a8ee, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,373 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=49f9e405ee146272624d3d205a4ebf9e, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,373 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=45e8fc1ea8df34f038aeb00318494097, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,374 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a06021a0c54a875234d29e87253d48a, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,374 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e8e6d28d8e1806fa7d958327b87df211, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,374 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=295cabe34d2d3dbccccb06cf3aa5bbf6, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,374 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bf73c409fe50a78c430f016d1431abfb, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,374 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-20T03:00:57,374 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732071717374 2024-11-20T03:00:57,374 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732071777374 2024-11-20T03:00:57,374 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-11-20T03:00:57,374 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-20T03:00:57,374 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T03:00:57,374 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T03:00:57,374 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T03:00:57,375 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-20T03:00:57,375 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-20T03:00:57,375 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-20T03:00:57,375 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T03:00:57,376 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-20T03:00:57,377 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-20T03:00:57,377 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T03:00:57,377 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T03:00:57,377 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T03:00:57,377 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T03:00:57,377 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T03:00:57,377 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T03:00:57,377 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T03:00:57,377 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T03:00:57,377 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T03:00:57,377 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T03:00:57,377 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T03:00:57,377 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T03:00:57,377 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T03:00:57,378 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T03:00:57,378 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-20T03:00:57,379 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf73c409fe50a78c430f016d1431abfb, server=1c82b83631a2,42975,1732071649704:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1c82b83631a2,43333,1732071649765, splitWal=true, meta=true 2024-11-20T03:00:57,379 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 1c82b83631a2%2C43333%2C1732071649765.meta.1732071651840.meta, worker=1c82b83631a2,36133,1732071649570 2024-11-20T03:00:57,379 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1c82b83631a2%2C43333%2C1732071649765.meta.1732071651840.meta, worker=1c82b83631a2,36133,1732071649570 2024-11-20T03:00:57,379 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T03:00:57,379 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1c82b83631a2,42975,1732071649704 2024-11-20T03:00:57,379 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 1c82b83631a2%2C43333%2C1732071649765.1732071651367, worker=1c82b83631a2,42975,1732071649704 2024-11-20T03:00:57,379 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1c82b83631a2%2C43333%2C1732071649765.1732071651367, worker=1c82b83631a2,42975,1732071649704 2024-11-20T03:00:57,379 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=38fa6330e381ad6ba7391bf9050c289e, ASSIGN 2024-11-20T03:00:57,379 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf73c409fe50a78c430f016d1431abfb, ASSIGN 2024-11-20T03:00:57,379 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ccd7aea6ec14848f8c6730c88ce1c780, ASSIGN 2024-11-20T03:00:57,380 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8a06021a0c54a875234d29e87253d48a, ASSIGN 2024-11-20T03:00:57,380 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4327d986b4dd716142908b96606b0ad7, ASSIGN 2024-11-20T03:00:57,380 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fc28de84807b58cb0d6ceaa73127debb, ASSIGN 2024-11-20T03:00:57,380 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8a818d7f8283ee7f2b5fae844b64a8ee, ASSIGN 2024-11-20T03:00:57,380 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf73c409fe50a78c430f016d1431abfb, server=1c82b83631a2,42975,1732071649704:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1c82b83631a2,43333,1732071649765, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:57,380 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 1c82b83631a2%2C43333%2C1732071649765.meta.1732071651840.meta, worker=1c82b83631a2,36133,1732071649570 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:57,380 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1c82b83631a2%2C43333%2C1732071649765.meta.1732071651840.meta, worker=1c82b83631a2,36133,1732071649570 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:57,380 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:57,380 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1c82b83631a2,42975,1732071649704 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:57,380 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 1c82b83631a2%2C43333%2C1732071649765.1732071651367, worker=1c82b83631a2,42975,1732071649704 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:57,380 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1c82b83631a2%2C43333%2C1732071649765.1732071651367, worker=1c82b83631a2,42975,1732071649704 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:57,380 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=38fa6330e381ad6ba7391bf9050c289e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:57,380 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=38fa6330e381ad6ba7391bf9050c289e, ASSIGN 2024-11-20T03:00:57,380 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf73c409fe50a78c430f016d1431abfb, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:57,380 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf73c409fe50a78c430f016d1431abfb, ASSIGN 2024-11-20T03:00:57,380 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ccd7aea6ec14848f8c6730c88ce1c780, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:57,380 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ccd7aea6ec14848f8c6730c88ce1c780, ASSIGN 2024-11-20T03:00:57,380 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8a06021a0c54a875234d29e87253d48a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:57,380 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8a06021a0c54a875234d29e87253d48a, ASSIGN 2024-11-20T03:00:57,380 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4327d986b4dd716142908b96606b0ad7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:57,380 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4327d986b4dd716142908b96606b0ad7, ASSIGN 2024-11-20T03:00:57,380 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fc28de84807b58cb0d6ceaa73127debb, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:57,380 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fc28de84807b58cb0d6ceaa73127debb, ASSIGN 2024-11-20T03:00:57,380 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8a818d7f8283ee7f2b5fae844b64a8ee, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:57,380 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8a818d7f8283ee7f2b5fae844b64a8ee, ASSIGN 2024-11-20T03:00:57,380 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-20T03:00:57,381 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=61 2024-11-20T03:00:57,381 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=1c82b83631a2,42975,1732071649704, table=hbase:meta, region=1588230740 2024-11-20T03:00:57,381 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1c82b83631a2,42975,1732071649704, state=OPEN 2024-11-20T03:00:57,390 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42975-0x10157797d910002, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T03:00:57,390 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36133-0x10157797d910001, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T03:00:57,390 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39617-0x10157797d910000, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T03:00:57,390 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T03:00:57,390 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T03:00:57,390 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T03:00:57,391 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=38fa6330e381ad6ba7391bf9050c289e, ASSIGN to state=OFFLINE, location=null, table=test, region=38fa6330e381ad6ba7391bf9050c289e to restore RIT 2024-11-20T03:00:57,391 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf73c409fe50a78c430f016d1431abfb, ASSIGN to state=OFFLINE, location=null, table=test, region=bf73c409fe50a78c430f016d1431abfb to restore RIT 2024-11-20T03:00:57,391 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ccd7aea6ec14848f8c6730c88ce1c780, ASSIGN to state=OFFLINE, location=null, table=test, region=ccd7aea6ec14848f8c6730c88ce1c780 to restore RIT 2024-11-20T03:00:57,391 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8a06021a0c54a875234d29e87253d48a, ASSIGN to state=OFFLINE, location=null, table=test, region=8a06021a0c54a875234d29e87253d48a to restore RIT 2024-11-20T03:00:57,391 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4327d986b4dd716142908b96606b0ad7, ASSIGN to state=OFFLINE, location=null, table=test, region=4327d986b4dd716142908b96606b0ad7 to restore RIT 2024-11-20T03:00:57,391 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fc28de84807b58cb0d6ceaa73127debb, ASSIGN to state=OFFLINE, location=null, table=test, region=fc28de84807b58cb0d6ceaa73127debb to restore RIT 2024-11-20T03:00:57,391 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8a818d7f8283ee7f2b5fae844b64a8ee, ASSIGN to state=OFFLINE, location=null, table=test, region=8a818d7f8283ee7f2b5fae844b64a8ee to restore RIT 2024-11-20T03:00:57,391 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732071687391 2024-11-20T03:00:57,391 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-20T03:00:57,391 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf73c409fe50a78c430f016d1431abfb, server=1c82b83631a2,42975,1732071649704:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1c82b83631a2,43333,1732071649765, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T03:00:57,393 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-20T03:00:57,395 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-20T03:00:57,395 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fc28de84807b58cb0d6ceaa73127debb, ASSIGN 2024-11-20T03:00:57,396 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T03:00:57,396 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fc28de84807b58cb0d6ceaa73127debb, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T03:00:57,396 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f63a07d19bc4399768e352bd9fcf3523, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,396 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=edd7cafc43290cb2489e8ba273907acd, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,396 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ccd7aea6ec14848f8c6730c88ce1c780, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,396 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=63b717291dd2cfc1f752c75b18ccc26b, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,397 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2fe87cdaf67461ff5c508d851e418c66, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,397 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fc28de84807b58cb0d6ceaa73127debb, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,397 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3c5e70743a336dce38e44ee0f8059378, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,397 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2816319e15be2a1534d61304db509629, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,397 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f928be6f5d7a3631384ebd87c8f80996, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,397 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad7c736680f4a75df049e62d21f77c48, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,397 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=56a64fab34378d9775e0bdf29862f7e6, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,397 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=535a88ef9d35d9795c43261c431087a8, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,398 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=38fa6330e381ad6ba7391bf9050c289e, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,398 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fbf7ef74de7c98db6b37604b08e67282, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,398 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4327d986b4dd716142908b96606b0ad7, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,398 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ef70776250de5f4e8ad7bd24027abe63, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,398 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=30cc99ff17ee8d21b793ce798b4998ba, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,398 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a641a5878ac82d20c96fa3dff0720791, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,399 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=550bbfe722915166ce74fec36828c508, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,399 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a818d7f8283ee7f2b5fae844b64a8ee, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,399 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=49f9e405ee146272624d3d205a4ebf9e, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,399 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=45e8fc1ea8df34f038aeb00318494097, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,399 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a06021a0c54a875234d29e87253d48a, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,399 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e8e6d28d8e1806fa7d958327b87df211, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,400 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=295cabe34d2d3dbccccb06cf3aa5bbf6, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,400 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-20T03:00:57,400 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bf73c409fe50a78c430f016d1431abfb, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,400 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-20T03:00:57,400 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732071717400 2024-11-20T03:00:57,400 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732071777400 2024-11-20T03:00:57,400 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-11-20T03:00:57,400 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-20T03:00:57,400 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T03:00:57,400 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T03:00:57,400 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T03:00:57,400 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-20T03:00:57,401 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-20T03:00:57,401 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-20T03:00:57,401 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T03:00:57,402 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-20T03:00:57,404 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-20T03:00:57,404 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T03:00:57,404 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T03:00:57,404 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T03:00:57,404 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T03:00:57,404 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T03:00:57,404 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T03:00:57,404 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T03:00:57,404 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T03:00:57,404 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T03:00:57,404 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T03:00:57,404 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T03:00:57,404 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T03:00:57,405 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T03:00:57,405 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-20T03:00:57,407 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf73c409fe50a78c430f016d1431abfb, server=1c82b83631a2,42975,1732071649704:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1c82b83631a2,43333,1732071649765, splitWal=true, meta=true 2024-11-20T03:00:57,407 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 1c82b83631a2%2C43333%2C1732071649765.meta.1732071651840.meta, worker=1c82b83631a2,36133,1732071649570 2024-11-20T03:00:57,407 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1c82b83631a2%2C43333%2C1732071649765.meta.1732071651840.meta, worker=1c82b83631a2,36133,1732071649570 2024-11-20T03:00:57,407 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T03:00:57,407 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1c82b83631a2,42975,1732071649704 2024-11-20T03:00:57,407 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 1c82b83631a2%2C43333%2C1732071649765.1732071651367, worker=1c82b83631a2,42975,1732071649704 2024-11-20T03:00:57,407 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1c82b83631a2%2C43333%2C1732071649765.1732071651367, worker=1c82b83631a2,42975,1732071649704 2024-11-20T03:00:57,407 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=38fa6330e381ad6ba7391bf9050c289e, ASSIGN 2024-11-20T03:00:57,407 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf73c409fe50a78c430f016d1431abfb, ASSIGN 2024-11-20T03:00:57,407 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ccd7aea6ec14848f8c6730c88ce1c780, ASSIGN 2024-11-20T03:00:57,407 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8a06021a0c54a875234d29e87253d48a, ASSIGN 2024-11-20T03:00:57,407 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4327d986b4dd716142908b96606b0ad7, ASSIGN 2024-11-20T03:00:57,407 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fc28de84807b58cb0d6ceaa73127debb, ASSIGN 2024-11-20T03:00:57,408 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf73c409fe50a78c430f016d1431abfb, server=1c82b83631a2,42975,1732071649704:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1c82b83631a2,43333,1732071649765, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:57,408 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 1c82b83631a2%2C43333%2C1732071649765.meta.1732071651840.meta, worker=1c82b83631a2,36133,1732071649570 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:57,408 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1c82b83631a2%2C43333%2C1732071649765.meta.1732071651840.meta, worker=1c82b83631a2,36133,1732071649570 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:57,408 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:57,408 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1c82b83631a2,42975,1732071649704 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:57,408 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 1c82b83631a2%2C43333%2C1732071649765.1732071651367, worker=1c82b83631a2,42975,1732071649704 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:57,408 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1c82b83631a2%2C43333%2C1732071649765.1732071651367, worker=1c82b83631a2,42975,1732071649704 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:57,408 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=38fa6330e381ad6ba7391bf9050c289e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:57,408 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=38fa6330e381ad6ba7391bf9050c289e, ASSIGN 2024-11-20T03:00:57,408 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf73c409fe50a78c430f016d1431abfb, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:57,408 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf73c409fe50a78c430f016d1431abfb, ASSIGN 2024-11-20T03:00:57,408 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ccd7aea6ec14848f8c6730c88ce1c780, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:57,408 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ccd7aea6ec14848f8c6730c88ce1c780, ASSIGN 2024-11-20T03:00:57,408 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8a06021a0c54a875234d29e87253d48a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:57,408 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8a06021a0c54a875234d29e87253d48a, ASSIGN 2024-11-20T03:00:57,408 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4327d986b4dd716142908b96606b0ad7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:57,408 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4327d986b4dd716142908b96606b0ad7, ASSIGN 2024-11-20T03:00:57,408 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fc28de84807b58cb0d6ceaa73127debb, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:57,408 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fc28de84807b58cb0d6ceaa73127debb, ASSIGN 2024-11-20T03:00:57,409 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-20T03:00:57,411 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=61 2024-11-20T03:00:57,411 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=1c82b83631a2,42975,1732071649704, table=hbase:meta, region=1588230740 2024-11-20T03:00:57,411 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1c82b83631a2,42975,1732071649704, state=OPEN 2024-11-20T03:00:57,421 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42975-0x10157797d910002, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T03:00:57,421 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36133-0x10157797d910001, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T03:00:57,421 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39617-0x10157797d910000, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T03:00:57,421 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T03:00:57,421 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T03:00:57,421 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T03:00:57,422 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=38fa6330e381ad6ba7391bf9050c289e, ASSIGN to state=OFFLINE, location=null, table=test, region=38fa6330e381ad6ba7391bf9050c289e to restore RIT 2024-11-20T03:00:57,422 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf73c409fe50a78c430f016d1431abfb, ASSIGN to state=OFFLINE, location=null, table=test, region=bf73c409fe50a78c430f016d1431abfb to restore RIT 2024-11-20T03:00:57,422 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ccd7aea6ec14848f8c6730c88ce1c780, ASSIGN to state=OFFLINE, location=null, table=test, region=ccd7aea6ec14848f8c6730c88ce1c780 to restore RIT 2024-11-20T03:00:57,422 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8a06021a0c54a875234d29e87253d48a, ASSIGN to state=OFFLINE, location=null, table=test, region=8a06021a0c54a875234d29e87253d48a to restore RIT 2024-11-20T03:00:57,423 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4327d986b4dd716142908b96606b0ad7, ASSIGN to state=OFFLINE, location=null, table=test, region=4327d986b4dd716142908b96606b0ad7 to restore RIT 2024-11-20T03:00:57,423 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fc28de84807b58cb0d6ceaa73127debb, ASSIGN to state=OFFLINE, location=null, table=test, region=fc28de84807b58cb0d6ceaa73127debb to restore RIT 2024-11-20T03:00:57,423 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732071687423 2024-11-20T03:00:57,423 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-20T03:00:57,423 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf73c409fe50a78c430f016d1431abfb, server=1c82b83631a2,42975,1732071649704:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1c82b83631a2,43333,1732071649765, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T03:00:57,424 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-20T03:00:57,427 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-20T03:00:57,427 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4327d986b4dd716142908b96606b0ad7, ASSIGN 2024-11-20T03:00:57,427 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T03:00:57,427 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4327d986b4dd716142908b96606b0ad7, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T03:00:57,429 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f63a07d19bc4399768e352bd9fcf3523, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,429 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=edd7cafc43290cb2489e8ba273907acd, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,430 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ccd7aea6ec14848f8c6730c88ce1c780, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,430 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=63b717291dd2cfc1f752c75b18ccc26b, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,430 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2fe87cdaf67461ff5c508d851e418c66, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,430 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fc28de84807b58cb0d6ceaa73127debb, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,430 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3c5e70743a336dce38e44ee0f8059378, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,430 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2816319e15be2a1534d61304db509629, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,430 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f928be6f5d7a3631384ebd87c8f80996, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,431 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad7c736680f4a75df049e62d21f77c48, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,431 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=56a64fab34378d9775e0bdf29862f7e6, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,431 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=535a88ef9d35d9795c43261c431087a8, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,431 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=38fa6330e381ad6ba7391bf9050c289e, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,431 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fbf7ef74de7c98db6b37604b08e67282, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,431 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4327d986b4dd716142908b96606b0ad7, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,431 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ef70776250de5f4e8ad7bd24027abe63, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,432 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=30cc99ff17ee8d21b793ce798b4998ba, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,432 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a641a5878ac82d20c96fa3dff0720791, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,432 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=550bbfe722915166ce74fec36828c508, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,432 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a818d7f8283ee7f2b5fae844b64a8ee, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,432 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=49f9e405ee146272624d3d205a4ebf9e, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,432 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=45e8fc1ea8df34f038aeb00318494097, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,432 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a06021a0c54a875234d29e87253d48a, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,433 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e8e6d28d8e1806fa7d958327b87df211, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,433 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=295cabe34d2d3dbccccb06cf3aa5bbf6, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,433 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bf73c409fe50a78c430f016d1431abfb, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,433 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-20T03:00:57,433 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732071717433 2024-11-20T03:00:57,433 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732071777433 2024-11-20T03:00:57,433 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 10 msec 2024-11-20T03:00:57,433 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-20T03:00:57,433 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T03:00:57,433 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T03:00:57,433 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T03:00:57,434 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-20T03:00:57,434 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-20T03:00:57,434 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-20T03:00:57,434 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T03:00:57,435 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-20T03:00:57,437 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-20T03:00:57,437 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T03:00:57,437 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T03:00:57,437 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T03:00:57,438 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T03:00:57,438 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T03:00:57,438 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T03:00:57,438 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T03:00:57,438 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T03:00:57,438 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T03:00:57,438 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T03:00:57,438 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T03:00:57,438 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T03:00:57,439 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-20T03:00:57,440 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf73c409fe50a78c430f016d1431abfb, server=1c82b83631a2,42975,1732071649704:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1c82b83631a2,43333,1732071649765, splitWal=true, meta=true 2024-11-20T03:00:57,440 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 1c82b83631a2%2C43333%2C1732071649765.meta.1732071651840.meta, worker=1c82b83631a2,36133,1732071649570 2024-11-20T03:00:57,440 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1c82b83631a2%2C43333%2C1732071649765.meta.1732071651840.meta, worker=1c82b83631a2,36133,1732071649570 2024-11-20T03:00:57,440 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T03:00:57,440 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1c82b83631a2,42975,1732071649704 2024-11-20T03:00:57,440 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 1c82b83631a2%2C43333%2C1732071649765.1732071651367, worker=1c82b83631a2,42975,1732071649704 2024-11-20T03:00:57,440 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1c82b83631a2%2C43333%2C1732071649765.1732071651367, worker=1c82b83631a2,42975,1732071649704 2024-11-20T03:00:57,440 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=38fa6330e381ad6ba7391bf9050c289e, ASSIGN 2024-11-20T03:00:57,441 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf73c409fe50a78c430f016d1431abfb, ASSIGN 2024-11-20T03:00:57,441 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ccd7aea6ec14848f8c6730c88ce1c780, ASSIGN 2024-11-20T03:00:57,441 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8a06021a0c54a875234d29e87253d48a, ASSIGN 2024-11-20T03:00:57,441 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4327d986b4dd716142908b96606b0ad7, ASSIGN 2024-11-20T03:00:57,441 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf73c409fe50a78c430f016d1431abfb, server=1c82b83631a2,42975,1732071649704:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1c82b83631a2,43333,1732071649765, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:57,441 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 1c82b83631a2%2C43333%2C1732071649765.meta.1732071651840.meta, worker=1c82b83631a2,36133,1732071649570 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:57,441 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1c82b83631a2%2C43333%2C1732071649765.meta.1732071651840.meta, worker=1c82b83631a2,36133,1732071649570 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:57,441 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:57,441 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1c82b83631a2,42975,1732071649704 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:57,441 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 1c82b83631a2%2C43333%2C1732071649765.1732071651367, worker=1c82b83631a2,42975,1732071649704 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:57,441 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1c82b83631a2%2C43333%2C1732071649765.1732071651367, worker=1c82b83631a2,42975,1732071649704 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:57,441 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=38fa6330e381ad6ba7391bf9050c289e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:57,441 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=38fa6330e381ad6ba7391bf9050c289e, ASSIGN 2024-11-20T03:00:57,441 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf73c409fe50a78c430f016d1431abfb, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:57,441 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf73c409fe50a78c430f016d1431abfb, ASSIGN 2024-11-20T03:00:57,441 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ccd7aea6ec14848f8c6730c88ce1c780, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:57,441 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ccd7aea6ec14848f8c6730c88ce1c780, ASSIGN 2024-11-20T03:00:57,441 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8a06021a0c54a875234d29e87253d48a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:57,441 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8a06021a0c54a875234d29e87253d48a, ASSIGN 2024-11-20T03:00:57,442 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4327d986b4dd716142908b96606b0ad7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:57,442 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4327d986b4dd716142908b96606b0ad7, ASSIGN 2024-11-20T03:00:57,442 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-20T03:00:57,443 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=61 2024-11-20T03:00:57,443 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=1c82b83631a2,42975,1732071649704, table=hbase:meta, region=1588230740 2024-11-20T03:00:57,443 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1c82b83631a2,42975,1732071649704, state=OPEN 2024-11-20T03:00:57,450 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39617 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T03:00:57,452 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39617 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T03:00:57,454 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'test' 2024-11-20T03:00:57,454 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42975-0x10157797d910002, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T03:00:57,454 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36133-0x10157797d910001, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T03:00:57,455 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39617-0x10157797d910000, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T03:00:57,455 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T03:00:57,455 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T03:00:57,455 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T03:00:57,456 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=38fa6330e381ad6ba7391bf9050c289e, ASSIGN to state=OFFLINE, location=null, table=test, region=38fa6330e381ad6ba7391bf9050c289e to restore RIT 2024-11-20T03:00:57,456 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf73c409fe50a78c430f016d1431abfb, ASSIGN to state=OFFLINE, location=null, table=test, region=bf73c409fe50a78c430f016d1431abfb to restore RIT 2024-11-20T03:00:57,456 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ccd7aea6ec14848f8c6730c88ce1c780, ASSIGN to state=OFFLINE, location=null, table=test, region=ccd7aea6ec14848f8c6730c88ce1c780 to restore RIT 2024-11-20T03:00:57,456 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8a06021a0c54a875234d29e87253d48a, ASSIGN to state=OFFLINE, location=null, table=test, region=8a06021a0c54a875234d29e87253d48a to restore RIT 2024-11-20T03:00:57,456 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4327d986b4dd716142908b96606b0ad7, ASSIGN to state=OFFLINE, location=null, table=test, region=4327d986b4dd716142908b96606b0ad7 to restore RIT 2024-11-20T03:00:57,457 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732071687457 2024-11-20T03:00:57,457 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-20T03:00:57,457 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf73c409fe50a78c430f016d1431abfb, server=1c82b83631a2,42975,1732071649704:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1c82b83631a2,43333,1732071649765, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T03:00:57,458 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-20T03:00:57,461 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-20T03:00:57,461 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8a06021a0c54a875234d29e87253d48a, ASSIGN 2024-11-20T03:00:57,461 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T03:00:57,461 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f63a07d19bc4399768e352bd9fcf3523, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,461 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8a06021a0c54a875234d29e87253d48a, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T03:00:57,461 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=edd7cafc43290cb2489e8ba273907acd, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,462 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ccd7aea6ec14848f8c6730c88ce1c780, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,462 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=63b717291dd2cfc1f752c75b18ccc26b, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,462 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2fe87cdaf67461ff5c508d851e418c66, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,462 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fc28de84807b58cb0d6ceaa73127debb, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,462 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3c5e70743a336dce38e44ee0f8059378, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,462 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2816319e15be2a1534d61304db509629, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,462 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f928be6f5d7a3631384ebd87c8f80996, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,462 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad7c736680f4a75df049e62d21f77c48, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,462 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=56a64fab34378d9775e0bdf29862f7e6, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,462 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=535a88ef9d35d9795c43261c431087a8, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,463 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=38fa6330e381ad6ba7391bf9050c289e, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,463 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fbf7ef74de7c98db6b37604b08e67282, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,463 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4327d986b4dd716142908b96606b0ad7, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,463 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ef70776250de5f4e8ad7bd24027abe63, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,463 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=30cc99ff17ee8d21b793ce798b4998ba, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,463 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a641a5878ac82d20c96fa3dff0720791, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,463 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=550bbfe722915166ce74fec36828c508, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,463 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a818d7f8283ee7f2b5fae844b64a8ee, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,463 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=49f9e405ee146272624d3d205a4ebf9e, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,463 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=45e8fc1ea8df34f038aeb00318494097, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,463 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a06021a0c54a875234d29e87253d48a, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,464 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e8e6d28d8e1806fa7d958327b87df211, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,464 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=295cabe34d2d3dbccccb06cf3aa5bbf6, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,464 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bf73c409fe50a78c430f016d1431abfb, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,464 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-20T03:00:57,464 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732071717464 2024-11-20T03:00:57,464 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732071777464 2024-11-20T03:00:57,464 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-11-20T03:00:57,464 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-20T03:00:57,464 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T03:00:57,464 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T03:00:57,464 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T03:00:57,464 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-20T03:00:57,464 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-20T03:00:57,464 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-20T03:00:57,465 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T03:00:57,465 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-20T03:00:57,467 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-20T03:00:57,467 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T03:00:57,467 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T03:00:57,467 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T03:00:57,467 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T03:00:57,467 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T03:00:57,467 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T03:00:57,467 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T03:00:57,467 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T03:00:57,467 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T03:00:57,467 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T03:00:57,467 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T03:00:57,468 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-20T03:00:57,468 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf73c409fe50a78c430f016d1431abfb, server=1c82b83631a2,42975,1732071649704:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1c82b83631a2,43333,1732071649765, splitWal=true, meta=true 2024-11-20T03:00:57,469 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 1c82b83631a2%2C43333%2C1732071649765.meta.1732071651840.meta, worker=1c82b83631a2,36133,1732071649570 2024-11-20T03:00:57,469 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1c82b83631a2%2C43333%2C1732071649765.meta.1732071651840.meta, worker=1c82b83631a2,36133,1732071649570 2024-11-20T03:00:57,469 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T03:00:57,469 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1c82b83631a2,42975,1732071649704 2024-11-20T03:00:57,469 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 1c82b83631a2%2C43333%2C1732071649765.1732071651367, worker=1c82b83631a2,42975,1732071649704 2024-11-20T03:00:57,469 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1c82b83631a2%2C43333%2C1732071649765.1732071651367, worker=1c82b83631a2,42975,1732071649704 2024-11-20T03:00:57,469 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=38fa6330e381ad6ba7391bf9050c289e, ASSIGN 2024-11-20T03:00:57,469 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf73c409fe50a78c430f016d1431abfb, ASSIGN 2024-11-20T03:00:57,469 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ccd7aea6ec14848f8c6730c88ce1c780, ASSIGN 2024-11-20T03:00:57,469 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8a06021a0c54a875234d29e87253d48a, ASSIGN 2024-11-20T03:00:57,469 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf73c409fe50a78c430f016d1431abfb, server=1c82b83631a2,42975,1732071649704:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1c82b83631a2,43333,1732071649765, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:57,469 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 1c82b83631a2%2C43333%2C1732071649765.meta.1732071651840.meta, worker=1c82b83631a2,36133,1732071649570 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:57,469 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1c82b83631a2%2C43333%2C1732071649765.meta.1732071651840.meta, worker=1c82b83631a2,36133,1732071649570 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:57,469 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:57,469 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1c82b83631a2,42975,1732071649704 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:57,469 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 1c82b83631a2%2C43333%2C1732071649765.1732071651367, worker=1c82b83631a2,42975,1732071649704 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:57,469 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1c82b83631a2%2C43333%2C1732071649765.1732071651367, worker=1c82b83631a2,42975,1732071649704 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:57,469 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=38fa6330e381ad6ba7391bf9050c289e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:57,469 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=38fa6330e381ad6ba7391bf9050c289e, ASSIGN 2024-11-20T03:00:57,469 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf73c409fe50a78c430f016d1431abfb, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:57,469 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf73c409fe50a78c430f016d1431abfb, ASSIGN 2024-11-20T03:00:57,469 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ccd7aea6ec14848f8c6730c88ce1c780, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:57,469 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ccd7aea6ec14848f8c6730c88ce1c780, ASSIGN 2024-11-20T03:00:57,469 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8a06021a0c54a875234d29e87253d48a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:57,469 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8a06021a0c54a875234d29e87253d48a, ASSIGN 2024-11-20T03:00:57,470 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-20T03:00:57,470 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=61 2024-11-20T03:00:57,470 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=1c82b83631a2,42975,1732071649704, table=hbase:meta, region=1588230740 2024-11-20T03:00:57,471 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1c82b83631a2,42975,1732071649704, state=OPEN 2024-11-20T03:00:57,481 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39617-0x10157797d910000, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T03:00:57,481 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42975-0x10157797d910002, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T03:00:57,481 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36133-0x10157797d910001, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T03:00:57,482 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T03:00:57,482 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T03:00:57,482 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T03:00:57,482 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=38fa6330e381ad6ba7391bf9050c289e, ASSIGN to state=OFFLINE, location=null, table=test, region=38fa6330e381ad6ba7391bf9050c289e to restore RIT 2024-11-20T03:00:57,482 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf73c409fe50a78c430f016d1431abfb, ASSIGN to state=OFFLINE, location=null, table=test, region=bf73c409fe50a78c430f016d1431abfb to restore RIT 2024-11-20T03:00:57,482 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ccd7aea6ec14848f8c6730c88ce1c780, ASSIGN to state=OFFLINE, location=null, table=test, region=ccd7aea6ec14848f8c6730c88ce1c780 to restore RIT 2024-11-20T03:00:57,482 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8a06021a0c54a875234d29e87253d48a, ASSIGN to state=OFFLINE, location=null, table=test, region=8a06021a0c54a875234d29e87253d48a to restore RIT 2024-11-20T03:00:57,483 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732071687483 2024-11-20T03:00:57,483 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-20T03:00:57,483 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf73c409fe50a78c430f016d1431abfb, server=1c82b83631a2,42975,1732071649704:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1c82b83631a2,43333,1732071649765, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T03:00:57,484 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-20T03:00:57,487 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-20T03:00:57,487 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ccd7aea6ec14848f8c6730c88ce1c780, ASSIGN 2024-11-20T03:00:57,487 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T03:00:57,487 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ccd7aea6ec14848f8c6730c88ce1c780, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T03:00:57,488 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f63a07d19bc4399768e352bd9fcf3523, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,488 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=edd7cafc43290cb2489e8ba273907acd, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,488 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ccd7aea6ec14848f8c6730c88ce1c780, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,488 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=63b717291dd2cfc1f752c75b18ccc26b, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,488 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2fe87cdaf67461ff5c508d851e418c66, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,488 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fc28de84807b58cb0d6ceaa73127debb, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,488 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3c5e70743a336dce38e44ee0f8059378, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,488 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2816319e15be2a1534d61304db509629, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,488 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f928be6f5d7a3631384ebd87c8f80996, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,489 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad7c736680f4a75df049e62d21f77c48, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,489 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=56a64fab34378d9775e0bdf29862f7e6, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,489 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=535a88ef9d35d9795c43261c431087a8, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,489 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=38fa6330e381ad6ba7391bf9050c289e, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,489 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fbf7ef74de7c98db6b37604b08e67282, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,489 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4327d986b4dd716142908b96606b0ad7, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,489 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ef70776250de5f4e8ad7bd24027abe63, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,489 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=30cc99ff17ee8d21b793ce798b4998ba, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,489 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a641a5878ac82d20c96fa3dff0720791, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,489 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=550bbfe722915166ce74fec36828c508, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,490 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a818d7f8283ee7f2b5fae844b64a8ee, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,490 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=49f9e405ee146272624d3d205a4ebf9e, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,490 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=45e8fc1ea8df34f038aeb00318494097, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,490 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a06021a0c54a875234d29e87253d48a, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,490 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e8e6d28d8e1806fa7d958327b87df211, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,490 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=295cabe34d2d3dbccccb06cf3aa5bbf6, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,490 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bf73c409fe50a78c430f016d1431abfb, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,490 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-20T03:00:57,490 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732071717490 2024-11-20T03:00:57,490 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732071777490 2024-11-20T03:00:57,490 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-11-20T03:00:57,490 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-20T03:00:57,490 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T03:00:57,490 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T03:00:57,490 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T03:00:57,491 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-20T03:00:57,491 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-20T03:00:57,491 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-20T03:00:57,491 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T03:00:57,492 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-20T03:00:57,493 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-20T03:00:57,493 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T03:00:57,493 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T03:00:57,493 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T03:00:57,493 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T03:00:57,493 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T03:00:57,493 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T03:00:57,493 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T03:00:57,493 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T03:00:57,493 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T03:00:57,493 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T03:00:57,494 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-20T03:00:57,495 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf73c409fe50a78c430f016d1431abfb, server=1c82b83631a2,42975,1732071649704:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1c82b83631a2,43333,1732071649765, splitWal=true, meta=true 2024-11-20T03:00:57,495 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 1c82b83631a2%2C43333%2C1732071649765.meta.1732071651840.meta, worker=1c82b83631a2,36133,1732071649570 2024-11-20T03:00:57,495 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1c82b83631a2%2C43333%2C1732071649765.meta.1732071651840.meta, worker=1c82b83631a2,36133,1732071649570 2024-11-20T03:00:57,495 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T03:00:57,495 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1c82b83631a2,42975,1732071649704 2024-11-20T03:00:57,495 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 1c82b83631a2%2C43333%2C1732071649765.1732071651367, worker=1c82b83631a2,42975,1732071649704 2024-11-20T03:00:57,495 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1c82b83631a2%2C43333%2C1732071649765.1732071651367, worker=1c82b83631a2,42975,1732071649704 2024-11-20T03:00:57,495 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=38fa6330e381ad6ba7391bf9050c289e, ASSIGN 2024-11-20T03:00:57,495 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf73c409fe50a78c430f016d1431abfb, ASSIGN 2024-11-20T03:00:57,495 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ccd7aea6ec14848f8c6730c88ce1c780, ASSIGN 2024-11-20T03:00:57,495 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf73c409fe50a78c430f016d1431abfb, server=1c82b83631a2,42975,1732071649704:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1c82b83631a2,43333,1732071649765, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:57,495 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 1c82b83631a2%2C43333%2C1732071649765.meta.1732071651840.meta, worker=1c82b83631a2,36133,1732071649570 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:57,495 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1c82b83631a2%2C43333%2C1732071649765.meta.1732071651840.meta, worker=1c82b83631a2,36133,1732071649570 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:57,495 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:57,495 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1c82b83631a2,42975,1732071649704 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:57,495 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 1c82b83631a2%2C43333%2C1732071649765.1732071651367, worker=1c82b83631a2,42975,1732071649704 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:57,495 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1c82b83631a2%2C43333%2C1732071649765.1732071651367, worker=1c82b83631a2,42975,1732071649704 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:57,495 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=38fa6330e381ad6ba7391bf9050c289e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:57,495 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=38fa6330e381ad6ba7391bf9050c289e, ASSIGN 2024-11-20T03:00:57,495 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf73c409fe50a78c430f016d1431abfb, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:57,495 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf73c409fe50a78c430f016d1431abfb, ASSIGN 2024-11-20T03:00:57,495 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ccd7aea6ec14848f8c6730c88ce1c780, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:57,495 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ccd7aea6ec14848f8c6730c88ce1c780, ASSIGN 2024-11-20T03:00:57,495 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-20T03:00:57,496 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=61 2024-11-20T03:00:57,496 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=1c82b83631a2,42975,1732071649704, table=hbase:meta, region=1588230740 2024-11-20T03:00:57,496 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1c82b83631a2,42975,1732071649704, state=OPEN 2024-11-20T03:00:57,506 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36133-0x10157797d910001, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T03:00:57,506 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42975-0x10157797d910002, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T03:00:57,507 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T03:00:57,507 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T03:00:57,507 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39617-0x10157797d910000, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T03:00:57,507 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T03:00:57,508 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=38fa6330e381ad6ba7391bf9050c289e, ASSIGN to state=OFFLINE, location=null, table=test, region=38fa6330e381ad6ba7391bf9050c289e to restore RIT 2024-11-20T03:00:57,508 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf73c409fe50a78c430f016d1431abfb, ASSIGN to state=OFFLINE, location=null, table=test, region=bf73c409fe50a78c430f016d1431abfb to restore RIT 2024-11-20T03:00:57,508 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ccd7aea6ec14848f8c6730c88ce1c780, ASSIGN to state=OFFLINE, location=null, table=test, region=ccd7aea6ec14848f8c6730c88ce1c780 to restore RIT 2024-11-20T03:00:57,509 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732071687509 2024-11-20T03:00:57,509 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-20T03:00:57,509 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf73c409fe50a78c430f016d1431abfb, server=1c82b83631a2,42975,1732071649704:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1c82b83631a2,43333,1732071649765, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T03:00:57,510 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-20T03:00:57,514 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-20T03:00:57,514 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf73c409fe50a78c430f016d1431abfb, ASSIGN 2024-11-20T03:00:57,514 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T03:00:57,514 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf73c409fe50a78c430f016d1431abfb, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T03:00:57,515 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f63a07d19bc4399768e352bd9fcf3523, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,515 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=edd7cafc43290cb2489e8ba273907acd, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,515 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ccd7aea6ec14848f8c6730c88ce1c780, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,515 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=63b717291dd2cfc1f752c75b18ccc26b, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,516 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2fe87cdaf67461ff5c508d851e418c66, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,516 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fc28de84807b58cb0d6ceaa73127debb, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,516 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3c5e70743a336dce38e44ee0f8059378, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,516 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2816319e15be2a1534d61304db509629, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,516 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f928be6f5d7a3631384ebd87c8f80996, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,516 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad7c736680f4a75df049e62d21f77c48, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,516 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=56a64fab34378d9775e0bdf29862f7e6, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,517 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=535a88ef9d35d9795c43261c431087a8, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,517 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=38fa6330e381ad6ba7391bf9050c289e, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,517 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fbf7ef74de7c98db6b37604b08e67282, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,517 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4327d986b4dd716142908b96606b0ad7, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,517 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ef70776250de5f4e8ad7bd24027abe63, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,517 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=30cc99ff17ee8d21b793ce798b4998ba, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,517 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a641a5878ac82d20c96fa3dff0720791, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,518 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=550bbfe722915166ce74fec36828c508, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,518 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a818d7f8283ee7f2b5fae844b64a8ee, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,518 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=49f9e405ee146272624d3d205a4ebf9e, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,518 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=45e8fc1ea8df34f038aeb00318494097, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,518 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a06021a0c54a875234d29e87253d48a, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,518 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e8e6d28d8e1806fa7d958327b87df211, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,519 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=295cabe34d2d3dbccccb06cf3aa5bbf6, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,519 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bf73c409fe50a78c430f016d1431abfb, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,519 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-20T03:00:57,519 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732071717519 2024-11-20T03:00:57,519 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732071777519 2024-11-20T03:00:57,519 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-11-20T03:00:57,519 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-20T03:00:57,519 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T03:00:57,519 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T03:00:57,519 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T03:00:57,520 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-20T03:00:57,520 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-20T03:00:57,520 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-20T03:00:57,520 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T03:00:57,522 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-20T03:00:57,523 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-20T03:00:57,523 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T03:00:57,523 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T03:00:57,523 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T03:00:57,524 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T03:00:57,524 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T03:00:57,524 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T03:00:57,524 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T03:00:57,524 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T03:00:57,524 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T03:00:57,524 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-20T03:00:57,525 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf73c409fe50a78c430f016d1431abfb, server=1c82b83631a2,42975,1732071649704:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1c82b83631a2,43333,1732071649765, splitWal=true, meta=true 2024-11-20T03:00:57,525 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 1c82b83631a2%2C43333%2C1732071649765.meta.1732071651840.meta, worker=1c82b83631a2,36133,1732071649570 2024-11-20T03:00:57,525 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1c82b83631a2%2C43333%2C1732071649765.meta.1732071651840.meta, worker=1c82b83631a2,36133,1732071649570 2024-11-20T03:00:57,525 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T03:00:57,525 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1c82b83631a2,42975,1732071649704 2024-11-20T03:00:57,525 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 1c82b83631a2%2C43333%2C1732071649765.1732071651367, worker=1c82b83631a2,42975,1732071649704 2024-11-20T03:00:57,525 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1c82b83631a2%2C43333%2C1732071649765.1732071651367, worker=1c82b83631a2,42975,1732071649704 2024-11-20T03:00:57,525 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=38fa6330e381ad6ba7391bf9050c289e, ASSIGN 2024-11-20T03:00:57,525 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf73c409fe50a78c430f016d1431abfb, ASSIGN 2024-11-20T03:00:57,525 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf73c409fe50a78c430f016d1431abfb, server=1c82b83631a2,42975,1732071649704:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1c82b83631a2,43333,1732071649765, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:57,525 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 1c82b83631a2%2C43333%2C1732071649765.meta.1732071651840.meta, worker=1c82b83631a2,36133,1732071649570 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:57,525 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1c82b83631a2%2C43333%2C1732071649765.meta.1732071651840.meta, worker=1c82b83631a2,36133,1732071649570 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:57,525 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:57,525 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1c82b83631a2,42975,1732071649704 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:57,526 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 1c82b83631a2%2C43333%2C1732071649765.1732071651367, worker=1c82b83631a2,42975,1732071649704 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:57,526 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1c82b83631a2%2C43333%2C1732071649765.1732071651367, worker=1c82b83631a2,42975,1732071649704 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:57,526 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=38fa6330e381ad6ba7391bf9050c289e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:57,526 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=38fa6330e381ad6ba7391bf9050c289e, ASSIGN 2024-11-20T03:00:57,526 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf73c409fe50a78c430f016d1431abfb, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:57,526 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf73c409fe50a78c430f016d1431abfb, ASSIGN 2024-11-20T03:00:57,526 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-20T03:00:57,527 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=61 2024-11-20T03:00:57,527 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=1c82b83631a2,42975,1732071649704, table=hbase:meta, region=1588230740 2024-11-20T03:00:57,527 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1c82b83631a2,42975,1732071649704, state=OPEN 2024-11-20T03:00:57,557 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39617 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T03:00:57,557 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39617 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T03:00:57,574 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39617-0x10157797d910000, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T03:00:57,574 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36133-0x10157797d910001, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T03:00:57,574 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42975-0x10157797d910002, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T03:00:57,575 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T03:00:57,575 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T03:00:57,575 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T03:00:57,576 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=38fa6330e381ad6ba7391bf9050c289e, ASSIGN to state=OFFLINE, location=null, table=test, region=38fa6330e381ad6ba7391bf9050c289e to restore RIT 2024-11-20T03:00:57,576 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf73c409fe50a78c430f016d1431abfb, ASSIGN to state=OFFLINE, location=null, table=test, region=bf73c409fe50a78c430f016d1431abfb to restore RIT 2024-11-20T03:00:57,578 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732071687578 2024-11-20T03:00:57,578 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-20T03:00:57,578 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf73c409fe50a78c430f016d1431abfb, server=1c82b83631a2,42975,1732071649704:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1c82b83631a2,43333,1732071649765, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T03:00:57,580 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-20T03:00:57,580 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-20T03:00:57,581 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=38fa6330e381ad6ba7391bf9050c289e, ASSIGN 2024-11-20T03:00:57,581 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T03:00:57,581 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=38fa6330e381ad6ba7391bf9050c289e, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T03:00:57,584 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f63a07d19bc4399768e352bd9fcf3523, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,584 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=edd7cafc43290cb2489e8ba273907acd, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,584 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ccd7aea6ec14848f8c6730c88ce1c780, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,584 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=63b717291dd2cfc1f752c75b18ccc26b, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,585 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2fe87cdaf67461ff5c508d851e418c66, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,585 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fc28de84807b58cb0d6ceaa73127debb, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,585 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3c5e70743a336dce38e44ee0f8059378, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,585 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2816319e15be2a1534d61304db509629, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,585 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f928be6f5d7a3631384ebd87c8f80996, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,585 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad7c736680f4a75df049e62d21f77c48, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,585 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=56a64fab34378d9775e0bdf29862f7e6, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,586 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=535a88ef9d35d9795c43261c431087a8, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,586 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=38fa6330e381ad6ba7391bf9050c289e, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,586 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fbf7ef74de7c98db6b37604b08e67282, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,586 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4327d986b4dd716142908b96606b0ad7, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,586 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ef70776250de5f4e8ad7bd24027abe63, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,586 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=30cc99ff17ee8d21b793ce798b4998ba, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,586 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a641a5878ac82d20c96fa3dff0720791, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,586 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=550bbfe722915166ce74fec36828c508, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,586 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a818d7f8283ee7f2b5fae844b64a8ee, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,587 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=49f9e405ee146272624d3d205a4ebf9e, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,587 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=45e8fc1ea8df34f038aeb00318494097, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,587 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a06021a0c54a875234d29e87253d48a, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,587 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e8e6d28d8e1806fa7d958327b87df211, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,587 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=295cabe34d2d3dbccccb06cf3aa5bbf6, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,587 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bf73c409fe50a78c430f016d1431abfb, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,587 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-20T03:00:57,587 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732071717587 2024-11-20T03:00:57,587 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732071777587 2024-11-20T03:00:57,588 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-11-20T03:00:57,588 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-20T03:00:57,588 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T03:00:57,588 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T03:00:57,588 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T03:00:57,588 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-20T03:00:57,588 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-20T03:00:57,588 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-20T03:00:57,588 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T03:00:57,589 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-20T03:00:57,591 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-20T03:00:57,591 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T03:00:57,591 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T03:00:57,591 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T03:00:57,591 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T03:00:57,591 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T03:00:57,591 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T03:00:57,592 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T03:00:57,592 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T03:00:57,593 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-20T03:00:57,594 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf73c409fe50a78c430f016d1431abfb, server=1c82b83631a2,42975,1732071649704:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1c82b83631a2,43333,1732071649765, splitWal=true, meta=true 2024-11-20T03:00:57,594 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 1c82b83631a2%2C43333%2C1732071649765.meta.1732071651840.meta, worker=1c82b83631a2,36133,1732071649570 2024-11-20T03:00:57,594 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1c82b83631a2%2C43333%2C1732071649765.meta.1732071651840.meta, worker=1c82b83631a2,36133,1732071649570 2024-11-20T03:00:57,594 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T03:00:57,594 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1c82b83631a2,42975,1732071649704 2024-11-20T03:00:57,594 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 1c82b83631a2%2C43333%2C1732071649765.1732071651367, worker=1c82b83631a2,42975,1732071649704 2024-11-20T03:00:57,594 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1c82b83631a2%2C43333%2C1732071649765.1732071651367, worker=1c82b83631a2,42975,1732071649704 2024-11-20T03:00:57,594 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=38fa6330e381ad6ba7391bf9050c289e, ASSIGN 2024-11-20T03:00:57,594 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf73c409fe50a78c430f016d1431abfb, server=1c82b83631a2,42975,1732071649704:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1c82b83631a2,43333,1732071649765, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:57,594 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 1c82b83631a2%2C43333%2C1732071649765.meta.1732071651840.meta, worker=1c82b83631a2,36133,1732071649570 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:57,595 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1c82b83631a2%2C43333%2C1732071649765.meta.1732071651840.meta, worker=1c82b83631a2,36133,1732071649570 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:57,595 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:57,595 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1c82b83631a2,42975,1732071649704 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:57,595 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 1c82b83631a2%2C43333%2C1732071649765.1732071651367, worker=1c82b83631a2,42975,1732071649704 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:57,595 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1c82b83631a2%2C43333%2C1732071649765.1732071651367, worker=1c82b83631a2,42975,1732071649704 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:57,595 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=38fa6330e381ad6ba7391bf9050c289e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:57,595 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=38fa6330e381ad6ba7391bf9050c289e, ASSIGN 2024-11-20T03:00:57,595 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-20T03:00:57,596 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=61 2024-11-20T03:00:57,596 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=1c82b83631a2,42975,1732071649704, table=hbase:meta, region=1588230740 2024-11-20T03:00:57,596 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1c82b83631a2,42975,1732071649704, state=OPEN 2024-11-20T03:00:57,615 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36133-0x10157797d910001, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T03:00:57,615 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42975-0x10157797d910002, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T03:00:57,615 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39617-0x10157797d910000, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T03:00:57,615 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T03:00:57,615 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T03:00:57,615 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T03:00:57,616 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=38fa6330e381ad6ba7391bf9050c289e, ASSIGN to state=OFFLINE, location=null, table=test, region=38fa6330e381ad6ba7391bf9050c289e to restore RIT 2024-11-20T03:00:57,616 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732071687616 2024-11-20T03:00:57,616 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-20T03:00:57,616 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf73c409fe50a78c430f016d1431abfb, server=1c82b83631a2,42975,1732071649704:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1c82b83631a2,43333,1732071649765, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T03:00:57,618 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-20T03:00:57,619 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-20T03:00:57,619 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1c82b83631a2%2C43333%2C1732071649765.1732071651367, worker=1c82b83631a2,42975,1732071649704 2024-11-20T03:00:57,619 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T03:00:57,619 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1c82b83631a2%2C43333%2C1732071649765.1732071651367, worker=1c82b83631a2,42975,1732071649704 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T03:00:57,621 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f63a07d19bc4399768e352bd9fcf3523, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,621 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=edd7cafc43290cb2489e8ba273907acd, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,621 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ccd7aea6ec14848f8c6730c88ce1c780, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,621 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=63b717291dd2cfc1f752c75b18ccc26b, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,621 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2fe87cdaf67461ff5c508d851e418c66, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,622 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fc28de84807b58cb0d6ceaa73127debb, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,622 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3c5e70743a336dce38e44ee0f8059378, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,622 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2816319e15be2a1534d61304db509629, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,622 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f928be6f5d7a3631384ebd87c8f80996, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,622 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad7c736680f4a75df049e62d21f77c48, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,622 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=56a64fab34378d9775e0bdf29862f7e6, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,622 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=535a88ef9d35d9795c43261c431087a8, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,622 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=38fa6330e381ad6ba7391bf9050c289e, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,623 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fbf7ef74de7c98db6b37604b08e67282, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,623 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4327d986b4dd716142908b96606b0ad7, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,623 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ef70776250de5f4e8ad7bd24027abe63, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,623 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=30cc99ff17ee8d21b793ce798b4998ba, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,623 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a641a5878ac82d20c96fa3dff0720791, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,623 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=550bbfe722915166ce74fec36828c508, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,623 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a818d7f8283ee7f2b5fae844b64a8ee, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,623 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=49f9e405ee146272624d3d205a4ebf9e, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,623 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=45e8fc1ea8df34f038aeb00318494097, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,624 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a06021a0c54a875234d29e87253d48a, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,624 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e8e6d28d8e1806fa7d958327b87df211, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,624 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=295cabe34d2d3dbccccb06cf3aa5bbf6, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,624 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bf73c409fe50a78c430f016d1431abfb, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,624 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-20T03:00:57,624 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732071717624 2024-11-20T03:00:57,624 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732071777624 2024-11-20T03:00:57,624 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-11-20T03:00:57,624 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-20T03:00:57,624 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T03:00:57,624 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T03:00:57,624 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T03:00:57,625 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-20T03:00:57,625 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-20T03:00:57,625 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-20T03:00:57,625 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T03:00:57,626 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-20T03:00:57,627 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-20T03:00:57,627 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T03:00:57,627 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T03:00:57,627 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T03:00:57,627 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T03:00:57,627 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T03:00:57,627 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T03:00:57,627 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T03:00:57,628 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-20T03:00:57,628 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf73c409fe50a78c430f016d1431abfb, server=1c82b83631a2,42975,1732071649704:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1c82b83631a2,43333,1732071649765, splitWal=true, meta=true 2024-11-20T03:00:57,628 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 1c82b83631a2%2C43333%2C1732071649765.meta.1732071651840.meta, worker=1c82b83631a2,36133,1732071649570 2024-11-20T03:00:57,629 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1c82b83631a2%2C43333%2C1732071649765.meta.1732071651840.meta, worker=1c82b83631a2,36133,1732071649570 2024-11-20T03:00:57,629 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T03:00:57,629 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1c82b83631a2,42975,1732071649704 2024-11-20T03:00:57,629 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 1c82b83631a2%2C43333%2C1732071649765.1732071651367, worker=1c82b83631a2,42975,1732071649704 2024-11-20T03:00:57,629 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1c82b83631a2%2C43333%2C1732071649765.1732071651367, worker=1c82b83631a2,42975,1732071649704 2024-11-20T03:00:57,629 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf73c409fe50a78c430f016d1431abfb, server=1c82b83631a2,42975,1732071649704:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1c82b83631a2,43333,1732071649765, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:57,629 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 1c82b83631a2%2C43333%2C1732071649765.meta.1732071651840.meta, worker=1c82b83631a2,36133,1732071649570 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:57,629 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1c82b83631a2%2C43333%2C1732071649765.meta.1732071651840.meta, worker=1c82b83631a2,36133,1732071649570 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:57,629 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:57,629 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1c82b83631a2,42975,1732071649704 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:57,629 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 1c82b83631a2%2C43333%2C1732071649765.1732071651367, worker=1c82b83631a2,42975,1732071649704 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:57,629 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1c82b83631a2%2C43333%2C1732071649765.1732071651367, worker=1c82b83631a2,42975,1732071649704 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:57,629 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-20T03:00:57,630 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=61 2024-11-20T03:00:57,630 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=1c82b83631a2,42975,1732071649704, table=hbase:meta, region=1588230740 2024-11-20T03:00:57,630 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1c82b83631a2,42975,1732071649704, state=OPEN 2024-11-20T03:00:57,640 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36133-0x10157797d910001, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T03:00:57,640 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39617-0x10157797d910000, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T03:00:57,640 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42975-0x10157797d910002, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T03:00:57,640 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T03:00:57,640 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T03:00:57,640 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T03:00:57,641 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732071687641 2024-11-20T03:00:57,641 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-20T03:00:57,641 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf73c409fe50a78c430f016d1431abfb, server=1c82b83631a2,42975,1732071649704:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1c82b83631a2,43333,1732071649765, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T03:00:57,642 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-20T03:00:57,643 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-20T03:00:57,643 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 1c82b83631a2%2C43333%2C1732071649765.1732071651367, worker=1c82b83631a2,42975,1732071649704 2024-11-20T03:00:57,643 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T03:00:57,643 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 1c82b83631a2%2C43333%2C1732071649765.1732071651367, worker=1c82b83631a2,42975,1732071649704 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T03:00:57,646 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f63a07d19bc4399768e352bd9fcf3523, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,646 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=edd7cafc43290cb2489e8ba273907acd, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,646 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ccd7aea6ec14848f8c6730c88ce1c780, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,646 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=63b717291dd2cfc1f752c75b18ccc26b, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,646 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2fe87cdaf67461ff5c508d851e418c66, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,646 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fc28de84807b58cb0d6ceaa73127debb, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,646 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3c5e70743a336dce38e44ee0f8059378, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,646 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2816319e15be2a1534d61304db509629, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,647 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f928be6f5d7a3631384ebd87c8f80996, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,647 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad7c736680f4a75df049e62d21f77c48, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,647 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=56a64fab34378d9775e0bdf29862f7e6, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,647 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=535a88ef9d35d9795c43261c431087a8, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,647 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=38fa6330e381ad6ba7391bf9050c289e, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,647 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fbf7ef74de7c98db6b37604b08e67282, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,647 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4327d986b4dd716142908b96606b0ad7, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,647 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ef70776250de5f4e8ad7bd24027abe63, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,647 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=30cc99ff17ee8d21b793ce798b4998ba, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,648 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a641a5878ac82d20c96fa3dff0720791, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,648 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=550bbfe722915166ce74fec36828c508, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,648 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a818d7f8283ee7f2b5fae844b64a8ee, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,648 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=49f9e405ee146272624d3d205a4ebf9e, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,648 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=45e8fc1ea8df34f038aeb00318494097, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,648 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a06021a0c54a875234d29e87253d48a, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,648 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e8e6d28d8e1806fa7d958327b87df211, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,648 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=295cabe34d2d3dbccccb06cf3aa5bbf6, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,648 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bf73c409fe50a78c430f016d1431abfb, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,648 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-20T03:00:57,648 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732071717648 2024-11-20T03:00:57,649 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732071777648 2024-11-20T03:00:57,649 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-11-20T03:00:57,649 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-20T03:00:57,649 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T03:00:57,649 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T03:00:57,649 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T03:00:57,649 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-20T03:00:57,649 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-20T03:00:57,649 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-20T03:00:57,650 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T03:00:57,651 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-20T03:00:57,653 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-20T03:00:57,653 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T03:00:57,653 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T03:00:57,653 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T03:00:57,653 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T03:00:57,653 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T03:00:57,653 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T03:00:57,654 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-20T03:00:57,654 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf73c409fe50a78c430f016d1431abfb, server=1c82b83631a2,42975,1732071649704:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1c82b83631a2,43333,1732071649765, splitWal=true, meta=true 2024-11-20T03:00:57,655 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 1c82b83631a2%2C43333%2C1732071649765.meta.1732071651840.meta, worker=1c82b83631a2,36133,1732071649570 2024-11-20T03:00:57,655 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1c82b83631a2%2C43333%2C1732071649765.meta.1732071651840.meta, worker=1c82b83631a2,36133,1732071649570 2024-11-20T03:00:57,655 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T03:00:57,655 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1c82b83631a2,42975,1732071649704 2024-11-20T03:00:57,655 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 1c82b83631a2%2C43333%2C1732071649765.1732071651367, worker=1c82b83631a2,42975,1732071649704 2024-11-20T03:00:57,655 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf73c409fe50a78c430f016d1431abfb, server=1c82b83631a2,42975,1732071649704:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1c82b83631a2,43333,1732071649765, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:57,655 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 1c82b83631a2%2C43333%2C1732071649765.meta.1732071651840.meta, worker=1c82b83631a2,36133,1732071649570 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:57,655 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1c82b83631a2%2C43333%2C1732071649765.meta.1732071651840.meta, worker=1c82b83631a2,36133,1732071649570 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:57,655 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:57,655 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1c82b83631a2,42975,1732071649704 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:57,655 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 1c82b83631a2%2C43333%2C1732071649765.1732071651367, worker=1c82b83631a2,42975,1732071649704 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:57,655 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-20T03:00:57,656 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=61 2024-11-20T03:00:57,656 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=1c82b83631a2,42975,1732071649704, table=hbase:meta, region=1588230740 2024-11-20T03:00:57,656 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1c82b83631a2,42975,1732071649704, state=OPEN 2024-11-20T03:00:57,665 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36133-0x10157797d910001, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T03:00:57,665 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42975-0x10157797d910002, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T03:00:57,665 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39617-0x10157797d910000, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T03:00:57,665 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T03:00:57,665 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T03:00:57,665 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T03:00:57,666 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732071687666 2024-11-20T03:00:57,666 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-20T03:00:57,666 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf73c409fe50a78c430f016d1431abfb, server=1c82b83631a2,42975,1732071649704:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1c82b83631a2,43333,1732071649765, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T03:00:57,667 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-20T03:00:57,668 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-20T03:00:57,668 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1c82b83631a2,42975,1732071649704 2024-11-20T03:00:57,668 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T03:00:57,669 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1c82b83631a2,42975,1732071649704 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T03:00:57,672 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f63a07d19bc4399768e352bd9fcf3523, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,672 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=edd7cafc43290cb2489e8ba273907acd, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,672 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ccd7aea6ec14848f8c6730c88ce1c780, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,672 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=63b717291dd2cfc1f752c75b18ccc26b, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,672 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2fe87cdaf67461ff5c508d851e418c66, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,672 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fc28de84807b58cb0d6ceaa73127debb, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,672 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3c5e70743a336dce38e44ee0f8059378, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,672 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2816319e15be2a1534d61304db509629, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,672 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f928be6f5d7a3631384ebd87c8f80996, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,672 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad7c736680f4a75df049e62d21f77c48, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,672 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=56a64fab34378d9775e0bdf29862f7e6, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,672 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=535a88ef9d35d9795c43261c431087a8, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,673 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=38fa6330e381ad6ba7391bf9050c289e, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,673 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fbf7ef74de7c98db6b37604b08e67282, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,673 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4327d986b4dd716142908b96606b0ad7, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,673 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ef70776250de5f4e8ad7bd24027abe63, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,673 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=30cc99ff17ee8d21b793ce798b4998ba, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,673 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a641a5878ac82d20c96fa3dff0720791, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,673 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=550bbfe722915166ce74fec36828c508, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,673 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a818d7f8283ee7f2b5fae844b64a8ee, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,673 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=49f9e405ee146272624d3d205a4ebf9e, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,673 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=45e8fc1ea8df34f038aeb00318494097, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,673 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a06021a0c54a875234d29e87253d48a, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,673 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e8e6d28d8e1806fa7d958327b87df211, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,673 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=295cabe34d2d3dbccccb06cf3aa5bbf6, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,674 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bf73c409fe50a78c430f016d1431abfb, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,674 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-20T03:00:57,674 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732071717674 2024-11-20T03:00:57,674 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732071777674 2024-11-20T03:00:57,674 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-11-20T03:00:57,674 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-20T03:00:57,674 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T03:00:57,674 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T03:00:57,674 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T03:00:57,674 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-20T03:00:57,674 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-20T03:00:57,674 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-20T03:00:57,674 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T03:00:57,675 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-20T03:00:57,677 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-20T03:00:57,677 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T03:00:57,677 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T03:00:57,677 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T03:00:57,677 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T03:00:57,677 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T03:00:57,677 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39617 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T03:00:57,677 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39617 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T03:00:57,678 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-20T03:00:57,678 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf73c409fe50a78c430f016d1431abfb, server=1c82b83631a2,42975,1732071649704:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1c82b83631a2,43333,1732071649765, splitWal=true, meta=true 2024-11-20T03:00:57,678 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 1c82b83631a2%2C43333%2C1732071649765.meta.1732071651840.meta, worker=1c82b83631a2,36133,1732071649570 2024-11-20T03:00:57,679 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1c82b83631a2%2C43333%2C1732071649765.meta.1732071651840.meta, worker=1c82b83631a2,36133,1732071649570 2024-11-20T03:00:57,679 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T03:00:57,679 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1c82b83631a2,42975,1732071649704 2024-11-20T03:00:57,679 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf73c409fe50a78c430f016d1431abfb, server=1c82b83631a2,42975,1732071649704:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1c82b83631a2,43333,1732071649765, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:57,679 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 1c82b83631a2%2C43333%2C1732071649765.meta.1732071651840.meta, worker=1c82b83631a2,36133,1732071649570 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:57,679 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1c82b83631a2%2C43333%2C1732071649765.meta.1732071651840.meta, worker=1c82b83631a2,36133,1732071649570 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:57,679 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:57,679 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1c82b83631a2,42975,1732071649704 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:57,679 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-20T03:00:57,680 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=61 2024-11-20T03:00:57,680 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=1c82b83631a2,42975,1732071649704, table=hbase:meta, region=1588230740 2024-11-20T03:00:57,680 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1c82b83631a2,42975,1732071649704, state=OPEN 2024-11-20T03:00:57,690 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39617-0x10157797d910000, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T03:00:57,690 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36133-0x10157797d910001, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T03:00:57,690 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42975-0x10157797d910002, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T03:00:57,690 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T03:00:57,690 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T03:00:57,690 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T03:00:57,691 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732071687691 2024-11-20T03:00:57,691 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-20T03:00:57,691 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf73c409fe50a78c430f016d1431abfb, server=1c82b83631a2,42975,1732071649704:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1c82b83631a2,43333,1732071649765, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T03:00:57,692 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-20T03:00:57,693 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-20T03:00:57,693 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T03:00:57,693 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T03:00:57,693 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T03:00:57,695 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f63a07d19bc4399768e352bd9fcf3523, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,695 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=edd7cafc43290cb2489e8ba273907acd, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,695 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ccd7aea6ec14848f8c6730c88ce1c780, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,695 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=63b717291dd2cfc1f752c75b18ccc26b, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,695 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2fe87cdaf67461ff5c508d851e418c66, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,696 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fc28de84807b58cb0d6ceaa73127debb, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,696 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3c5e70743a336dce38e44ee0f8059378, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,696 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2816319e15be2a1534d61304db509629, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,696 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f928be6f5d7a3631384ebd87c8f80996, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,696 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad7c736680f4a75df049e62d21f77c48, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,696 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=56a64fab34378d9775e0bdf29862f7e6, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,696 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=535a88ef9d35d9795c43261c431087a8, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,696 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=38fa6330e381ad6ba7391bf9050c289e, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,696 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fbf7ef74de7c98db6b37604b08e67282, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,696 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4327d986b4dd716142908b96606b0ad7, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,696 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ef70776250de5f4e8ad7bd24027abe63, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,696 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=30cc99ff17ee8d21b793ce798b4998ba, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,697 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a641a5878ac82d20c96fa3dff0720791, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,697 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=550bbfe722915166ce74fec36828c508, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,697 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a818d7f8283ee7f2b5fae844b64a8ee, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,697 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=49f9e405ee146272624d3d205a4ebf9e, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,697 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=45e8fc1ea8df34f038aeb00318494097, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,697 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a06021a0c54a875234d29e87253d48a, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,697 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e8e6d28d8e1806fa7d958327b87df211, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,697 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=295cabe34d2d3dbccccb06cf3aa5bbf6, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,697 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bf73c409fe50a78c430f016d1431abfb, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,697 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-20T03:00:57,697 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732071717697 2024-11-20T03:00:57,697 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732071777697 2024-11-20T03:00:57,697 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 6 msec 2024-11-20T03:00:57,697 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-20T03:00:57,697 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T03:00:57,697 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T03:00:57,697 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T03:00:57,698 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-20T03:00:57,698 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-20T03:00:57,698 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-20T03:00:57,698 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T03:00:57,699 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-20T03:00:57,700 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-20T03:00:57,700 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T03:00:57,700 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T03:00:57,700 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T03:00:57,700 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T03:00:57,701 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-20T03:00:57,701 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf73c409fe50a78c430f016d1431abfb, server=1c82b83631a2,42975,1732071649704:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1c82b83631a2,43333,1732071649765, splitWal=true, meta=true 2024-11-20T03:00:57,701 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 1c82b83631a2%2C43333%2C1732071649765.meta.1732071651840.meta, worker=1c82b83631a2,36133,1732071649570 2024-11-20T03:00:57,701 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1c82b83631a2%2C43333%2C1732071649765.meta.1732071651840.meta, worker=1c82b83631a2,36133,1732071649570 2024-11-20T03:00:57,701 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T03:00:57,701 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf73c409fe50a78c430f016d1431abfb, server=1c82b83631a2,42975,1732071649704:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1c82b83631a2,43333,1732071649765, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:57,701 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 1c82b83631a2%2C43333%2C1732071649765.meta.1732071651840.meta, worker=1c82b83631a2,36133,1732071649570 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:57,701 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1c82b83631a2%2C43333%2C1732071649765.meta.1732071651840.meta, worker=1c82b83631a2,36133,1732071649570 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:57,701 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:57,701 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-11-20T03:00:57,702 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=61 2024-11-20T03:00:57,702 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=1c82b83631a2,42975,1732071649704, table=hbase:meta, region=1588230740 2024-11-20T03:00:57,702 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1c82b83631a2,42975,1732071649704, state=OPEN 2024-11-20T03:00:57,713 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42975-0x10157797d910002, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T03:00:57,713 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36133-0x10157797d910001, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T03:00:57,713 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39617-0x10157797d910000, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T03:00:57,713 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T03:00:57,713 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T03:00:57,713 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T03:00:57,714 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732071687714 2024-11-20T03:00:57,714 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-20T03:00:57,714 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf73c409fe50a78c430f016d1431abfb, server=1c82b83631a2,42975,1732071649704:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1c82b83631a2,43333,1732071649765, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T03:00:57,715 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-20T03:00:57,716 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-20T03:00:57,716 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1c82b83631a2%2C43333%2C1732071649765.meta.1732071651840.meta, worker=1c82b83631a2,36133,1732071649570 2024-11-20T03:00:57,716 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T03:00:57,716 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1c82b83631a2%2C43333%2C1732071649765.meta.1732071651840.meta, worker=1c82b83631a2,36133,1732071649570 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T03:00:57,719 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f63a07d19bc4399768e352bd9fcf3523, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,719 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=edd7cafc43290cb2489e8ba273907acd, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,719 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ccd7aea6ec14848f8c6730c88ce1c780, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,719 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=63b717291dd2cfc1f752c75b18ccc26b, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,719 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2fe87cdaf67461ff5c508d851e418c66, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,719 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fc28de84807b58cb0d6ceaa73127debb, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,720 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3c5e70743a336dce38e44ee0f8059378, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,720 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2816319e15be2a1534d61304db509629, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,720 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f928be6f5d7a3631384ebd87c8f80996, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,720 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad7c736680f4a75df049e62d21f77c48, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,720 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=56a64fab34378d9775e0bdf29862f7e6, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,720 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=535a88ef9d35d9795c43261c431087a8, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,720 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=38fa6330e381ad6ba7391bf9050c289e, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,720 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fbf7ef74de7c98db6b37604b08e67282, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,720 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4327d986b4dd716142908b96606b0ad7, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,720 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ef70776250de5f4e8ad7bd24027abe63, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,720 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=30cc99ff17ee8d21b793ce798b4998ba, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,720 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a641a5878ac82d20c96fa3dff0720791, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,720 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=550bbfe722915166ce74fec36828c508, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,721 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a818d7f8283ee7f2b5fae844b64a8ee, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,721 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=49f9e405ee146272624d3d205a4ebf9e, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,721 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=45e8fc1ea8df34f038aeb00318494097, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,721 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a06021a0c54a875234d29e87253d48a, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,721 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e8e6d28d8e1806fa7d958327b87df211, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,721 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=295cabe34d2d3dbccccb06cf3aa5bbf6, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,721 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bf73c409fe50a78c430f016d1431abfb, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,721 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-20T03:00:57,721 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732071717721 2024-11-20T03:00:57,721 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732071777721 2024-11-20T03:00:57,721 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-11-20T03:00:57,721 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-20T03:00:57,721 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T03:00:57,721 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T03:00:57,721 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T03:00:57,722 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-20T03:00:57,722 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-20T03:00:57,722 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-20T03:00:57,722 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T03:00:57,723 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-20T03:00:57,724 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-20T03:00:57,724 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T03:00:57,724 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T03:00:57,724 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T03:00:57,724 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-20T03:00:57,725 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf73c409fe50a78c430f016d1431abfb, server=1c82b83631a2,42975,1732071649704:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1c82b83631a2,43333,1732071649765, splitWal=true, meta=true 2024-11-20T03:00:57,725 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 1c82b83631a2%2C43333%2C1732071649765.meta.1732071651840.meta, worker=1c82b83631a2,36133,1732071649570 2024-11-20T03:00:57,725 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1c82b83631a2%2C43333%2C1732071649765.meta.1732071651840.meta, worker=1c82b83631a2,36133,1732071649570 2024-11-20T03:00:57,725 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf73c409fe50a78c430f016d1431abfb, server=1c82b83631a2,42975,1732071649704:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1c82b83631a2,43333,1732071649765, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:57,725 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 1c82b83631a2%2C43333%2C1732071649765.meta.1732071651840.meta, worker=1c82b83631a2,36133,1732071649570 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:57,725 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1c82b83631a2%2C43333%2C1732071649765.meta.1732071651840.meta, worker=1c82b83631a2,36133,1732071649570 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:57,725 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-11-20T03:00:57,726 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=61 2024-11-20T03:00:57,726 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=1c82b83631a2,42975,1732071649704, table=hbase:meta, region=1588230740 2024-11-20T03:00:57,726 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1c82b83631a2,42975,1732071649704, state=OPEN 2024-11-20T03:00:57,731 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39617-0x10157797d910000, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T03:00:57,731 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42975-0x10157797d910002, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T03:00:57,731 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36133-0x10157797d910001, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T03:00:57,732 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T03:00:57,732 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T03:00:57,732 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T03:00:57,732 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732071687732 2024-11-20T03:00:57,733 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-20T03:00:57,732 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf73c409fe50a78c430f016d1431abfb, server=1c82b83631a2,42975,1732071649704:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1c82b83631a2,43333,1732071649765, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T03:00:57,733 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-20T03:00:57,736 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-20T03:00:57,737 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 1c82b83631a2%2C43333%2C1732071649765.meta.1732071651840.meta, worker=1c82b83631a2,36133,1732071649570 2024-11-20T03:00:57,737 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T03:00:57,737 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f63a07d19bc4399768e352bd9fcf3523, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,737 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 1c82b83631a2%2C43333%2C1732071649765.meta.1732071651840.meta, worker=1c82b83631a2,36133,1732071649570 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T03:00:57,737 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=edd7cafc43290cb2489e8ba273907acd, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,737 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ccd7aea6ec14848f8c6730c88ce1c780, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,737 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=63b717291dd2cfc1f752c75b18ccc26b, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,737 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2fe87cdaf67461ff5c508d851e418c66, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,737 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fc28de84807b58cb0d6ceaa73127debb, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,737 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3c5e70743a336dce38e44ee0f8059378, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,738 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2816319e15be2a1534d61304db509629, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,738 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f928be6f5d7a3631384ebd87c8f80996, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,738 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad7c736680f4a75df049e62d21f77c48, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,738 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=56a64fab34378d9775e0bdf29862f7e6, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,738 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=535a88ef9d35d9795c43261c431087a8, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,738 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=38fa6330e381ad6ba7391bf9050c289e, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,738 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fbf7ef74de7c98db6b37604b08e67282, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,738 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4327d986b4dd716142908b96606b0ad7, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,738 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ef70776250de5f4e8ad7bd24027abe63, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,738 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=30cc99ff17ee8d21b793ce798b4998ba, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,739 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a641a5878ac82d20c96fa3dff0720791, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,739 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=550bbfe722915166ce74fec36828c508, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,739 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a818d7f8283ee7f2b5fae844b64a8ee, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,739 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=49f9e405ee146272624d3d205a4ebf9e, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,739 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=45e8fc1ea8df34f038aeb00318494097, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,739 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a06021a0c54a875234d29e87253d48a, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,739 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e8e6d28d8e1806fa7d958327b87df211, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,739 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=295cabe34d2d3dbccccb06cf3aa5bbf6, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,739 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bf73c409fe50a78c430f016d1431abfb, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,739 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-20T03:00:57,739 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732071717739 2024-11-20T03:00:57,740 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732071777739 2024-11-20T03:00:57,740 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-11-20T03:00:57,740 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-20T03:00:57,740 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T03:00:57,740 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T03:00:57,740 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T03:00:57,740 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-20T03:00:57,740 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-20T03:00:57,740 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-20T03:00:57,740 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T03:00:57,741 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-20T03:00:57,743 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-20T03:00:57,743 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T03:00:57,743 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T03:00:57,743 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-20T03:00:57,744 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf73c409fe50a78c430f016d1431abfb, server=1c82b83631a2,42975,1732071649704:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1c82b83631a2,43333,1732071649765, splitWal=true, meta=true 2024-11-20T03:00:57,744 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 1c82b83631a2%2C43333%2C1732071649765.meta.1732071651840.meta, worker=1c82b83631a2,36133,1732071649570 2024-11-20T03:00:57,744 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf73c409fe50a78c430f016d1431abfb, server=1c82b83631a2,42975,1732071649704:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1c82b83631a2,43333,1732071649765, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:57,744 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 1c82b83631a2%2C43333%2C1732071649765.meta.1732071651840.meta, worker=1c82b83631a2,36133,1732071649570 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T03:00:57,744 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-11-20T03:00:57,745 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=61 2024-11-20T03:00:57,745 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=1c82b83631a2,42975,1732071649704, table=hbase:meta, region=1588230740 2024-11-20T03:00:57,745 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1c82b83631a2,42975,1732071649704, state=OPEN 2024-11-20T03:00:57,757 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39617-0x10157797d910000, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T03:00:57,757 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42975-0x10157797d910002, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T03:00:57,757 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36133-0x10157797d910001, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T03:00:57,757 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T03:00:57,757 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T03:00:57,757 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T03:00:57,759 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732071687759 2024-11-20T03:00:57,759 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-20T03:00:57,759 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf73c409fe50a78c430f016d1431abfb, server=1c82b83631a2,42975,1732071649704:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1c82b83631a2,43333,1732071649765, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T03:00:57,760 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-20T03:00:57,762 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-20T03:00:57,762 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf73c409fe50a78c430f016d1431abfb, server=1c82b83631a2,42975,1732071649704:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1c82b83631a2,43333,1732071649765, splitWal=true, meta=true 2024-11-20T03:00:57,762 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T03:00:57,762 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf73c409fe50a78c430f016d1431abfb, server=1c82b83631a2,42975,1732071649704:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1c82b83631a2,43333,1732071649765, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1604) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T03:00:57,765 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f63a07d19bc4399768e352bd9fcf3523, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,765 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=edd7cafc43290cb2489e8ba273907acd, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,765 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ccd7aea6ec14848f8c6730c88ce1c780, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,765 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=63b717291dd2cfc1f752c75b18ccc26b, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,765 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2fe87cdaf67461ff5c508d851e418c66, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,765 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fc28de84807b58cb0d6ceaa73127debb, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,765 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3c5e70743a336dce38e44ee0f8059378, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,765 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2816319e15be2a1534d61304db509629, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,765 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f928be6f5d7a3631384ebd87c8f80996, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,766 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad7c736680f4a75df049e62d21f77c48, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,766 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=56a64fab34378d9775e0bdf29862f7e6, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,766 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=535a88ef9d35d9795c43261c431087a8, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,766 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=38fa6330e381ad6ba7391bf9050c289e, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,766 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fbf7ef74de7c98db6b37604b08e67282, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,766 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4327d986b4dd716142908b96606b0ad7, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,766 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ef70776250de5f4e8ad7bd24027abe63, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,766 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=30cc99ff17ee8d21b793ce798b4998ba, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,766 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a641a5878ac82d20c96fa3dff0720791, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,766 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=550bbfe722915166ce74fec36828c508, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,766 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a818d7f8283ee7f2b5fae844b64a8ee, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,766 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=49f9e405ee146272624d3d205a4ebf9e, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,766 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=45e8fc1ea8df34f038aeb00318494097, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,766 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a06021a0c54a875234d29e87253d48a, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,767 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e8e6d28d8e1806fa7d958327b87df211, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,767 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=295cabe34d2d3dbccccb06cf3aa5bbf6, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,767 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bf73c409fe50a78c430f016d1431abfb, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,767 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-20T03:00:57,767 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732071717767 2024-11-20T03:00:57,767 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732071777767 2024-11-20T03:00:57,767 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-11-20T03:00:57,767 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-20T03:00:57,767 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T03:00:57,767 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T03:00:57,767 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T03:00:57,767 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-20T03:00:57,767 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-20T03:00:57,767 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-20T03:00:57,768 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T03:00:57,768 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-20T03:00:57,769 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-20T03:00:57,770 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T03:00:57,770 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-20T03:00:57,770 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf73c409fe50a78c430f016d1431abfb, server=1c82b83631a2,42975,1732071649704:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1c82b83631a2,43333,1732071649765, splitWal=true, meta=true 2024-11-20T03:00:57,770 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf73c409fe50a78c430f016d1431abfb, server=1c82b83631a2,42975,1732071649704:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1c82b83631a2,43333,1732071649765, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-20T03:00:57,770 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-11-20T03:00:57,771 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=61 2024-11-20T03:00:57,771 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=1c82b83631a2,42975,1732071649704, table=hbase:meta, region=1588230740 2024-11-20T03:00:57,771 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1c82b83631a2,42975,1732071649704, state=OPEN 2024-11-20T03:00:57,780 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39617 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T03:00:57,781 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39617 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T03:00:57,781 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39617-0x10157797d910000, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T03:00:57,781 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42975-0x10157797d910002, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T03:00:57,781 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36133-0x10157797d910001, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T03:00:57,781 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T03:00:57,781 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T03:00:57,781 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T03:00:57,782 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732071687782 2024-11-20T03:00:57,782 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-20T03:00:57,782 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf73c409fe50a78c430f016d1431abfb, server=1c82b83631a2,42975,1732071649704:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1c82b83631a2,43333,1732071649765, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T03:00:57,783 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-20T03:00:57,788 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f63a07d19bc4399768e352bd9fcf3523, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,788 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=edd7cafc43290cb2489e8ba273907acd, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,788 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ccd7aea6ec14848f8c6730c88ce1c780, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,788 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=57, state=ROLLEDBACK, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf73c409fe50a78c430f016d1431abfb, server=1c82b83631a2,42975,1732071649704:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1c82b83631a2,43333,1732071649765, splitWal=true, meta=true exec-time=2.9570 sec 2024-11-20T03:00:57,788 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=63b717291dd2cfc1f752c75b18ccc26b, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,788 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2fe87cdaf67461ff5c508d851e418c66, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,788 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fc28de84807b58cb0d6ceaa73127debb, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,788 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3c5e70743a336dce38e44ee0f8059378, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,789 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2816319e15be2a1534d61304db509629, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,789 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f928be6f5d7a3631384ebd87c8f80996, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,789 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad7c736680f4a75df049e62d21f77c48, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,789 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=56a64fab34378d9775e0bdf29862f7e6, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,789 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=535a88ef9d35d9795c43261c431087a8, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,789 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=38fa6330e381ad6ba7391bf9050c289e, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,789 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fbf7ef74de7c98db6b37604b08e67282, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,789 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4327d986b4dd716142908b96606b0ad7, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,789 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ef70776250de5f4e8ad7bd24027abe63, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,790 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=30cc99ff17ee8d21b793ce798b4998ba, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,790 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a641a5878ac82d20c96fa3dff0720791, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,790 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=550bbfe722915166ce74fec36828c508, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,790 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a818d7f8283ee7f2b5fae844b64a8ee, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,790 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=49f9e405ee146272624d3d205a4ebf9e, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,790 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=45e8fc1ea8df34f038aeb00318494097, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,790 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a06021a0c54a875234d29e87253d48a, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,790 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e8e6d28d8e1806fa7d958327b87df211, regionState=OPEN, lastHost=1c82b83631a2,42975,1732071649704, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,790 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=295cabe34d2d3dbccccb06cf3aa5bbf6, regionState=OPEN, lastHost=1c82b83631a2,36133,1732071649570, regionLocation=1c82b83631a2,36133,1732071649570, openSeqNum=2 2024-11-20T03:00:57,791 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bf73c409fe50a78c430f016d1431abfb, regionState=OPENING, lastHost=1c82b83631a2,43333,1732071649765, regionLocation=1c82b83631a2,42975,1732071649704, openSeqNum=2 2024-11-20T03:00:57,791 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-20T03:00:57,791 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732071717791 2024-11-20T03:00:57,791 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732071777791 2024-11-20T03:00:57,791 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-11-20T03:00:57,808 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: master.assignment.TestRollbackSCP#testFailAndRollback Thread=348 (was 382), OpenFileDescriptor=618 (was 641), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=207 (was 190) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6376 (was 6440) 2024-11-20T03:00:57,808 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-20T03:00:57,809 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-20T03:00:57,809 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.tearDownAfterClass(TestRollbackSCP.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T03:00:57,809 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T03:00:57,810 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T03:00:57,810 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T03:00:57,810 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-20T03:00:57,810 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1636518704, stopped=false 2024-11-20T03:00:57,811 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=1c82b83631a2,39617,1732071648799 2024-11-20T03:00:57,821 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42975-0x10157797d910002, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-20T03:00:57,821 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39617-0x10157797d910000, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-20T03:00:57,821 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39617-0x10157797d910000, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T03:00:57,821 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42975-0x10157797d910002, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T03:00:57,821 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36133-0x10157797d910001, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-20T03:00:57,821 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36133-0x10157797d910001, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T03:00:57,821 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T03:00:57,821 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-20T03:00:57,822 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.tearDownAfterClass(TestRollbackSCP.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T03:00:57,822 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T03:00:57,822 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:42975-0x10157797d910002, quorum=127.0.0.1:52318, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T03:00:57,822 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '1c82b83631a2,36133,1732071649570' ***** 2024-11-20T03:00:57,822 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-20T03:00:57,822 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '1c82b83631a2,42975,1732071649704' ***** 2024-11-20T03:00:57,822 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-20T03:00:57,822 INFO [RS:0;1c82b83631a2:36133 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-20T03:00:57,822 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-20T03:00:57,822 INFO [RS:1;1c82b83631a2:42975 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-20T03:00:57,823 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-20T03:00:57,823 INFO [RS:0;1c82b83631a2:36133 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-20T03:00:57,823 INFO [RS:0;1c82b83631a2:36133 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-20T03:00:57,823 INFO [RS:1;1c82b83631a2:42975 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-20T03:00:57,823 INFO [RS:1;1c82b83631a2:42975 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-20T03:00:57,823 INFO [RS:0;1c82b83631a2:36133 {}] regionserver.HRegionServer(3091): Received CLOSE for 56a64fab34378d9775e0bdf29862f7e6 2024-11-20T03:00:57,823 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:39617-0x10157797d910000, quorum=127.0.0.1:52318, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T03:00:57,823 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36133-0x10157797d910001, quorum=127.0.0.1:52318, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T03:00:57,823 INFO [RS:1;1c82b83631a2:42975 {}] regionserver.HRegionServer(3091): Received CLOSE for f63a07d19bc4399768e352bd9fcf3523 2024-11-20T03:00:57,824 INFO [RS:0;1c82b83631a2:36133 {}] regionserver.HRegionServer(3091): Received CLOSE for 2816319e15be2a1534d61304db509629 2024-11-20T03:00:57,824 INFO [RS:0;1c82b83631a2:36133 {}] regionserver.HRegionServer(3091): Received CLOSE for 8a06021a0c54a875234d29e87253d48a 2024-11-20T03:00:57,824 INFO [RS:0;1c82b83631a2:36133 {}] regionserver.HRegionServer(3091): Received CLOSE for 535a88ef9d35d9795c43261c431087a8 2024-11-20T03:00:57,824 INFO [RS:0;1c82b83631a2:36133 {}] regionserver.HRegionServer(3091): Received CLOSE for edd7cafc43290cb2489e8ba273907acd 2024-11-20T03:00:57,824 INFO [RS:0;1c82b83631a2:36133 {}] regionserver.HRegionServer(3091): Received CLOSE for a641a5878ac82d20c96fa3dff0720791 2024-11-20T03:00:57,824 INFO [RS:0;1c82b83631a2:36133 {}] regionserver.HRegionServer(3091): Received CLOSE for 295cabe34d2d3dbccccb06cf3aa5bbf6 2024-11-20T03:00:57,824 INFO [RS:0;1c82b83631a2:36133 {}] regionserver.HRegionServer(3091): Received CLOSE for ccd7aea6ec14848f8c6730c88ce1c780 2024-11-20T03:00:57,824 INFO [RS:0;1c82b83631a2:36133 {}] regionserver.HRegionServer(3091): Received CLOSE for f928be6f5d7a3631384ebd87c8f80996 2024-11-20T03:00:57,824 INFO [RS:0;1c82b83631a2:36133 {}] regionserver.HRegionServer(3091): Received CLOSE for 4327d986b4dd716142908b96606b0ad7 2024-11-20T03:00:57,824 INFO [RS:0;1c82b83631a2:36133 {}] regionserver.HRegionServer(3091): Received CLOSE for 550bbfe722915166ce74fec36828c508 2024-11-20T03:00:57,824 INFO [RS:0;1c82b83631a2:36133 {}] regionserver.HRegionServer(3091): Received CLOSE for 2fe87cdaf67461ff5c508d851e418c66 2024-11-20T03:00:57,824 INFO [RS:0;1c82b83631a2:36133 {}] regionserver.HRegionServer(3091): Received CLOSE for 45e8fc1ea8df34f038aeb00318494097 2024-11-20T03:00:57,824 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 56a64fab34378d9775e0bdf29862f7e6, disabling compactions & flushes 2024-11-20T03:00:57,824 INFO [RS:0;1c82b83631a2:36133 {}] regionserver.HRegionServer(959): stopping server 1c82b83631a2,36133,1732071649570 2024-11-20T03:00:57,824 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,kkk,1732071652347.56a64fab34378d9775e0bdf29862f7e6. 2024-11-20T03:00:57,825 INFO [RS:0;1c82b83631a2:36133 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-20T03:00:57,825 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,kkk,1732071652347.56a64fab34378d9775e0bdf29862f7e6. 2024-11-20T03:00:57,825 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,kkk,1732071652347.56a64fab34378d9775e0bdf29862f7e6. after waiting 0 ms 2024-11-20T03:00:57,825 INFO [RS:0;1c82b83631a2:36133 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;1c82b83631a2:36133. 2024-11-20T03:00:57,825 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,kkk,1732071652347.56a64fab34378d9775e0bdf29862f7e6. 2024-11-20T03:00:57,825 DEBUG [RS:0;1c82b83631a2:36133 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T03:00:57,825 DEBUG [RS:0;1c82b83631a2:36133 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T03:00:57,825 INFO [RS:0;1c82b83631a2:36133 {}] regionserver.HRegionServer(1321): Waiting on 13 regions to close 2024-11-20T03:00:57,825 DEBUG [RS:0;1c82b83631a2:36133 {}] regionserver.HRegionServer(1351): Waiting on 2816319e15be2a1534d61304db509629, 295cabe34d2d3dbccccb06cf3aa5bbf6, 2fe87cdaf67461ff5c508d851e418c66, 4327d986b4dd716142908b96606b0ad7, 45e8fc1ea8df34f038aeb00318494097, 535a88ef9d35d9795c43261c431087a8, 550bbfe722915166ce74fec36828c508, 56a64fab34378d9775e0bdf29862f7e6, 8a06021a0c54a875234d29e87253d48a, a641a5878ac82d20c96fa3dff0720791, ccd7aea6ec14848f8c6730c88ce1c780, edd7cafc43290cb2489e8ba273907acd, f928be6f5d7a3631384ebd87c8f80996 2024-11-20T03:00:57,826 INFO [RS:1;1c82b83631a2:42975 {}] regionserver.HRegionServer(3091): Received CLOSE for bf73c409fe50a78c430f016d1431abfb 2024-11-20T03:00:57,826 INFO [RS:1;1c82b83631a2:42975 {}] regionserver.HRegionServer(3091): Received CLOSE for 30cc99ff17ee8d21b793ce798b4998ba 2024-11-20T03:00:57,826 INFO [RS:1;1c82b83631a2:42975 {}] regionserver.HRegionServer(3091): Received CLOSE for 49f9e405ee146272624d3d205a4ebf9e 2024-11-20T03:00:57,826 INFO [RS:1;1c82b83631a2:42975 {}] regionserver.HRegionServer(3091): Received CLOSE for 63b717291dd2cfc1f752c75b18ccc26b 2024-11-20T03:00:57,826 INFO [RS:1;1c82b83631a2:42975 {}] regionserver.HRegionServer(3091): Received CLOSE for ad7c736680f4a75df049e62d21f77c48 2024-11-20T03:00:57,826 INFO [RS:1;1c82b83631a2:42975 {}] regionserver.HRegionServer(3091): Received CLOSE for fc28de84807b58cb0d6ceaa73127debb 2024-11-20T03:00:57,826 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing f63a07d19bc4399768e352bd9fcf3523, disabling compactions & flushes 2024-11-20T03:00:57,826 INFO [RS:1;1c82b83631a2:42975 {}] regionserver.HRegionServer(3091): Received CLOSE for 8a818d7f8283ee7f2b5fae844b64a8ee 2024-11-20T03:00:57,826 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,,1732071652347.f63a07d19bc4399768e352bd9fcf3523. 2024-11-20T03:00:57,826 INFO [RS:1;1c82b83631a2:42975 {}] regionserver.HRegionServer(3091): Received CLOSE for ef70776250de5f4e8ad7bd24027abe63 2024-11-20T03:00:57,826 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,,1732071652347.f63a07d19bc4399768e352bd9fcf3523. 2024-11-20T03:00:57,826 INFO [RS:1;1c82b83631a2:42975 {}] regionserver.HRegionServer(3091): Received CLOSE for 38fa6330e381ad6ba7391bf9050c289e 2024-11-20T03:00:57,826 INFO [RS:1;1c82b83631a2:42975 {}] regionserver.HRegionServer(3091): Received CLOSE for e8e6d28d8e1806fa7d958327b87df211 2024-11-20T03:00:57,826 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,,1732071652347.f63a07d19bc4399768e352bd9fcf3523. after waiting 0 ms 2024-11-20T03:00:57,826 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,,1732071652347.f63a07d19bc4399768e352bd9fcf3523. 2024-11-20T03:00:57,826 INFO [RS:1;1c82b83631a2:42975 {}] regionserver.HRegionServer(3091): Received CLOSE for fbf7ef74de7c98db6b37604b08e67282 2024-11-20T03:00:57,826 INFO [RS:1;1c82b83631a2:42975 {}] regionserver.HRegionServer(3091): Received CLOSE for 3c5e70743a336dce38e44ee0f8059378 2024-11-20T03:00:57,826 INFO [RS:1;1c82b83631a2:42975 {}] regionserver.HRegionServer(959): stopping server 1c82b83631a2,42975,1732071649704 2024-11-20T03:00:57,826 INFO [RS:1;1c82b83631a2:42975 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-20T03:00:57,826 INFO [RS:1;1c82b83631a2:42975 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;1c82b83631a2:42975. 2024-11-20T03:00:57,827 DEBUG [RS:1;1c82b83631a2:42975 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T03:00:57,827 DEBUG [RS:1;1c82b83631a2:42975 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T03:00:57,827 INFO [RS:1;1c82b83631a2:42975 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-20T03:00:57,827 INFO [RS:1;1c82b83631a2:42975 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-20T03:00:57,827 INFO [RS:1;1c82b83631a2:42975 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-20T03:00:57,827 INFO [RS:1;1c82b83631a2:42975 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-20T03:00:57,834 INFO [RS:1;1c82b83631a2:42975 {}] regionserver.HRegionServer(1321): Waiting on 14 regions to close 2024-11-20T03:00:57,835 DEBUG [RS:1;1c82b83631a2:42975 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 30cc99ff17ee8d21b793ce798b4998ba, 38fa6330e381ad6ba7391bf9050c289e, 3c5e70743a336dce38e44ee0f8059378, 49f9e405ee146272624d3d205a4ebf9e, 63b717291dd2cfc1f752c75b18ccc26b, 8a818d7f8283ee7f2b5fae844b64a8ee, ad7c736680f4a75df049e62d21f77c48, bf73c409fe50a78c430f016d1431abfb, e8e6d28d8e1806fa7d958327b87df211, ef70776250de5f4e8ad7bd24027abe63, f63a07d19bc4399768e352bd9fcf3523, fbf7ef74de7c98db6b37604b08e67282, fc28de84807b58cb0d6ceaa73127debb 2024-11-20T03:00:57,835 DEBUG [RS_CLOSE_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-20T03:00:57,835 INFO [RS_CLOSE_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-20T03:00:57,835 DEBUG [RS_CLOSE_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-20T03:00:57,835 DEBUG [RS_CLOSE_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-20T03:00:57,835 DEBUG [RS_CLOSE_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-20T03:00:57,835 INFO [RS_CLOSE_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=2.65 KB heapSize=5.44 KB 2024-11-20T03:00:57,836 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/default/test/56a64fab34378d9775e0bdf29862f7e6/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T03:00:57,838 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,kkk,1732071652347.56a64fab34378d9775e0bdf29862f7e6. 2024-11-20T03:00:57,838 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 56a64fab34378d9775e0bdf29862f7e6: Waiting for close lock at 1732071657824Running coprocessor pre-close hooks at 1732071657824Disabling compacts and flushes for region at 1732071657824Disabling writes for close at 1732071657825 (+1 ms)Writing region close event to WAL at 1732071657825Running coprocessor post-close hooks at 1732071657836 (+11 ms)Closed at 1732071657838 (+2 ms) 2024-11-20T03:00:57,839 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/default/test/f63a07d19bc4399768e352bd9fcf3523/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T03:00:57,839 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,kkk,1732071652347.56a64fab34378d9775e0bdf29862f7e6. 2024-11-20T03:00:57,840 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 2816319e15be2a1534d61304db509629, disabling compactions & flushes 2024-11-20T03:00:57,840 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,hhh,1732071652347.2816319e15be2a1534d61304db509629. 2024-11-20T03:00:57,840 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,hhh,1732071652347.2816319e15be2a1534d61304db509629. 2024-11-20T03:00:57,840 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,hhh,1732071652347.2816319e15be2a1534d61304db509629. after waiting 0 ms 2024-11-20T03:00:57,840 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,hhh,1732071652347.2816319e15be2a1534d61304db509629. 2024-11-20T03:00:57,840 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,,1732071652347.f63a07d19bc4399768e352bd9fcf3523. 2024-11-20T03:00:57,840 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for f63a07d19bc4399768e352bd9fcf3523: Waiting for close lock at 1732071657826Running coprocessor pre-close hooks at 1732071657826Disabling compacts and flushes for region at 1732071657826Disabling writes for close at 1732071657826Writing region close event to WAL at 1732071657831 (+5 ms)Running coprocessor post-close hooks at 1732071657840 (+9 ms)Closed at 1732071657840 2024-11-20T03:00:57,840 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,,1732071652347.f63a07d19bc4399768e352bd9fcf3523. 2024-11-20T03:00:57,840 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing bf73c409fe50a78c430f016d1431abfb, disabling compactions & flushes 2024-11-20T03:00:57,840 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,zzz,1732071652347.bf73c409fe50a78c430f016d1431abfb. 2024-11-20T03:00:57,840 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,zzz,1732071652347.bf73c409fe50a78c430f016d1431abfb. 2024-11-20T03:00:57,840 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,zzz,1732071652347.bf73c409fe50a78c430f016d1431abfb. after waiting 0 ms 2024-11-20T03:00:57,840 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,zzz,1732071652347.bf73c409fe50a78c430f016d1431abfb. 2024-11-20T03:00:57,845 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/default/test/2816319e15be2a1534d61304db509629/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T03:00:57,846 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,hhh,1732071652347.2816319e15be2a1534d61304db509629. 2024-11-20T03:00:57,846 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 2816319e15be2a1534d61304db509629: Waiting for close lock at 1732071657839Running coprocessor pre-close hooks at 1732071657839Disabling compacts and flushes for region at 1732071657839Disabling writes for close at 1732071657840 (+1 ms)Writing region close event to WAL at 1732071657840Running coprocessor post-close hooks at 1732071657846 (+6 ms)Closed at 1732071657846 2024-11-20T03:00:57,846 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,hhh,1732071652347.2816319e15be2a1534d61304db509629. 2024-11-20T03:00:57,846 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 8a06021a0c54a875234d29e87253d48a, disabling compactions & flushes 2024-11-20T03:00:57,846 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,www,1732071652347.8a06021a0c54a875234d29e87253d48a. 2024-11-20T03:00:57,847 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,www,1732071652347.8a06021a0c54a875234d29e87253d48a. 2024-11-20T03:00:57,847 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,www,1732071652347.8a06021a0c54a875234d29e87253d48a. after waiting 0 ms 2024-11-20T03:00:57,847 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,www,1732071652347.8a06021a0c54a875234d29e87253d48a. 2024-11-20T03:00:57,848 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/default/test/bf73c409fe50a78c430f016d1431abfb/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T03:00:57,849 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,zzz,1732071652347.bf73c409fe50a78c430f016d1431abfb. 2024-11-20T03:00:57,849 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for bf73c409fe50a78c430f016d1431abfb: Waiting for close lock at 1732071657840Running coprocessor pre-close hooks at 1732071657840Disabling compacts and flushes for region at 1732071657840Disabling writes for close at 1732071657840Writing region close event to WAL at 1732071657843 (+3 ms)Running coprocessor post-close hooks at 1732071657849 (+6 ms)Closed at 1732071657849 2024-11-20T03:00:57,849 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,zzz,1732071652347.bf73c409fe50a78c430f016d1431abfb. 2024-11-20T03:00:57,850 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 30cc99ff17ee8d21b793ce798b4998ba, disabling compactions & flushes 2024-11-20T03:00:57,850 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,qqq,1732071652347.30cc99ff17ee8d21b793ce798b4998ba. 2024-11-20T03:00:57,850 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,qqq,1732071652347.30cc99ff17ee8d21b793ce798b4998ba. 2024-11-20T03:00:57,850 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,qqq,1732071652347.30cc99ff17ee8d21b793ce798b4998ba. after waiting 0 ms 2024-11-20T03:00:57,850 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,qqq,1732071652347.30cc99ff17ee8d21b793ce798b4998ba. 2024-11-20T03:00:57,851 DEBUG [RS_CLOSE_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/hbase/meta/1588230740/.tmp/info/40d0cc0c57104bc6a92ac4d998695d40 is 138, key is test,ccc,1732071652347.ccd7aea6ec14848f8c6730c88ce1c780./info:regioninfo/1732071656329/Put/seqid=0 2024-11-20T03:00:57,851 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/default/test/8a06021a0c54a875234d29e87253d48a/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T03:00:57,852 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,www,1732071652347.8a06021a0c54a875234d29e87253d48a. 2024-11-20T03:00:57,852 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 8a06021a0c54a875234d29e87253d48a: Waiting for close lock at 1732071657846Running coprocessor pre-close hooks at 1732071657846Disabling compacts and flushes for region at 1732071657846Disabling writes for close at 1732071657847 (+1 ms)Writing region close event to WAL at 1732071657847Running coprocessor post-close hooks at 1732071657852 (+5 ms)Closed at 1732071657852 2024-11-20T03:00:57,852 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,www,1732071652347.8a06021a0c54a875234d29e87253d48a. 2024-11-20T03:00:57,852 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 535a88ef9d35d9795c43261c431087a8, disabling compactions & flushes 2024-11-20T03:00:57,852 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,lll,1732071652347.535a88ef9d35d9795c43261c431087a8. 2024-11-20T03:00:57,852 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,lll,1732071652347.535a88ef9d35d9795c43261c431087a8. 2024-11-20T03:00:57,852 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,lll,1732071652347.535a88ef9d35d9795c43261c431087a8. after waiting 0 ms 2024-11-20T03:00:57,852 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,lll,1732071652347.535a88ef9d35d9795c43261c431087a8. 2024-11-20T03:00:57,856 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/default/test/30cc99ff17ee8d21b793ce798b4998ba/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T03:00:57,856 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,qqq,1732071652347.30cc99ff17ee8d21b793ce798b4998ba. 2024-11-20T03:00:57,857 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 30cc99ff17ee8d21b793ce798b4998ba: Waiting for close lock at 1732071657850Running coprocessor pre-close hooks at 1732071657850Disabling compacts and flushes for region at 1732071657850Disabling writes for close at 1732071657850Writing region close event to WAL at 1732071657850Running coprocessor post-close hooks at 1732071657856 (+6 ms)Closed at 1732071657856 2024-11-20T03:00:57,857 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,qqq,1732071652347.30cc99ff17ee8d21b793ce798b4998ba. 2024-11-20T03:00:57,857 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 49f9e405ee146272624d3d205a4ebf9e, disabling compactions & flushes 2024-11-20T03:00:57,857 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,uuu,1732071652347.49f9e405ee146272624d3d205a4ebf9e. 2024-11-20T03:00:57,857 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,uuu,1732071652347.49f9e405ee146272624d3d205a4ebf9e. 2024-11-20T03:00:57,857 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,uuu,1732071652347.49f9e405ee146272624d3d205a4ebf9e. after waiting 0 ms 2024-11-20T03:00:57,857 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,uuu,1732071652347.49f9e405ee146272624d3d205a4ebf9e. 2024-11-20T03:00:57,861 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/default/test/535a88ef9d35d9795c43261c431087a8/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T03:00:57,862 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,lll,1732071652347.535a88ef9d35d9795c43261c431087a8. 2024-11-20T03:00:57,862 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 535a88ef9d35d9795c43261c431087a8: Waiting for close lock at 1732071657852Running coprocessor pre-close hooks at 1732071657852Disabling compacts and flushes for region at 1732071657852Disabling writes for close at 1732071657852Writing region close event to WAL at 1732071657855 (+3 ms)Running coprocessor post-close hooks at 1732071657862 (+7 ms)Closed at 1732071657862 2024-11-20T03:00:57,862 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,lll,1732071652347.535a88ef9d35d9795c43261c431087a8. 2024-11-20T03:00:57,863 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing edd7cafc43290cb2489e8ba273907acd, disabling compactions & flushes 2024-11-20T03:00:57,863 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,bbb,1732071652347.edd7cafc43290cb2489e8ba273907acd. 2024-11-20T03:00:57,863 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,bbb,1732071652347.edd7cafc43290cb2489e8ba273907acd. 2024-11-20T03:00:57,863 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,bbb,1732071652347.edd7cafc43290cb2489e8ba273907acd. after waiting 0 ms 2024-11-20T03:00:57,863 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,bbb,1732071652347.edd7cafc43290cb2489e8ba273907acd. 2024-11-20T03:00:57,867 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/default/test/49f9e405ee146272624d3d205a4ebf9e/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T03:00:57,868 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,uuu,1732071652347.49f9e405ee146272624d3d205a4ebf9e. 2024-11-20T03:00:57,869 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 49f9e405ee146272624d3d205a4ebf9e: Waiting for close lock at 1732071657857Running coprocessor pre-close hooks at 1732071657857Disabling compacts and flushes for region at 1732071657857Disabling writes for close at 1732071657857Writing region close event to WAL at 1732071657857Running coprocessor post-close hooks at 1732071657868 (+11 ms)Closed at 1732071657868 2024-11-20T03:00:57,869 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,uuu,1732071652347.49f9e405ee146272624d3d205a4ebf9e. 2024-11-20T03:00:57,869 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 63b717291dd2cfc1f752c75b18ccc26b, disabling compactions & flushes 2024-11-20T03:00:57,869 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ddd,1732071652347.63b717291dd2cfc1f752c75b18ccc26b. 2024-11-20T03:00:57,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38873 is added to blk_1073741869_1045 (size=8192) 2024-11-20T03:00:57,869 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ddd,1732071652347.63b717291dd2cfc1f752c75b18ccc26b. 2024-11-20T03:00:57,869 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ddd,1732071652347.63b717291dd2cfc1f752c75b18ccc26b. after waiting 0 ms 2024-11-20T03:00:57,869 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ddd,1732071652347.63b717291dd2cfc1f752c75b18ccc26b. 2024-11-20T03:00:57,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42923 is added to blk_1073741869_1045 (size=8192) 2024-11-20T03:00:57,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33951 is added to blk_1073741869_1045 (size=8192) 2024-11-20T03:00:57,871 INFO [RS_CLOSE_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.65 KB at sequenceid=71 (bloomFilter=true), to=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/hbase/meta/1588230740/.tmp/info/40d0cc0c57104bc6a92ac4d998695d40 2024-11-20T03:00:57,876 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/default/test/edd7cafc43290cb2489e8ba273907acd/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T03:00:57,877 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/default/test/63b717291dd2cfc1f752c75b18ccc26b/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T03:00:57,877 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,bbb,1732071652347.edd7cafc43290cb2489e8ba273907acd. 2024-11-20T03:00:57,877 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for edd7cafc43290cb2489e8ba273907acd: Waiting for close lock at 1732071657863Running coprocessor pre-close hooks at 1732071657863Disabling compacts and flushes for region at 1732071657863Disabling writes for close at 1732071657863Writing region close event to WAL at 1732071657871 (+8 ms)Running coprocessor post-close hooks at 1732071657877 (+6 ms)Closed at 1732071657877 2024-11-20T03:00:57,877 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,bbb,1732071652347.edd7cafc43290cb2489e8ba273907acd. 2024-11-20T03:00:57,877 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ddd,1732071652347.63b717291dd2cfc1f752c75b18ccc26b. 2024-11-20T03:00:57,877 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 63b717291dd2cfc1f752c75b18ccc26b: Waiting for close lock at 1732071657869Running coprocessor pre-close hooks at 1732071657869Disabling compacts and flushes for region at 1732071657869Disabling writes for close at 1732071657869Writing region close event to WAL at 1732071657872 (+3 ms)Running coprocessor post-close hooks at 1732071657877 (+5 ms)Closed at 1732071657877 2024-11-20T03:00:57,877 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing a641a5878ac82d20c96fa3dff0720791, disabling compactions & flushes 2024-11-20T03:00:57,877 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,rrr,1732071652347.a641a5878ac82d20c96fa3dff0720791. 2024-11-20T03:00:57,877 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,rrr,1732071652347.a641a5878ac82d20c96fa3dff0720791. 2024-11-20T03:00:57,877 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ddd,1732071652347.63b717291dd2cfc1f752c75b18ccc26b. 2024-11-20T03:00:57,878 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,rrr,1732071652347.a641a5878ac82d20c96fa3dff0720791. after waiting 0 ms 2024-11-20T03:00:57,878 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,rrr,1732071652347.a641a5878ac82d20c96fa3dff0720791. 2024-11-20T03:00:57,878 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing ad7c736680f4a75df049e62d21f77c48, disabling compactions & flushes 2024-11-20T03:00:57,878 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,jjj,1732071652347.ad7c736680f4a75df049e62d21f77c48. 2024-11-20T03:00:57,878 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,jjj,1732071652347.ad7c736680f4a75df049e62d21f77c48. 2024-11-20T03:00:57,878 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,jjj,1732071652347.ad7c736680f4a75df049e62d21f77c48. after waiting 0 ms 2024-11-20T03:00:57,878 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,jjj,1732071652347.ad7c736680f4a75df049e62d21f77c48. 2024-11-20T03:00:57,881 DEBUG [RS_CLOSE_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/hbase/meta/1588230740/.tmp/info/40d0cc0c57104bc6a92ac4d998695d40 as hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/hbase/meta/1588230740/info/40d0cc0c57104bc6a92ac4d998695d40 2024-11-20T03:00:57,883 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/default/test/ad7c736680f4a75df049e62d21f77c48/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T03:00:57,884 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,jjj,1732071652347.ad7c736680f4a75df049e62d21f77c48. 2024-11-20T03:00:57,884 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/default/test/a641a5878ac82d20c96fa3dff0720791/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T03:00:57,884 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for ad7c736680f4a75df049e62d21f77c48: Waiting for close lock at 1732071657878Running coprocessor pre-close hooks at 1732071657878Disabling compacts and flushes for region at 1732071657878Disabling writes for close at 1732071657878Writing region close event to WAL at 1732071657878Running coprocessor post-close hooks at 1732071657884 (+6 ms)Closed at 1732071657884 2024-11-20T03:00:57,884 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,jjj,1732071652347.ad7c736680f4a75df049e62d21f77c48. 2024-11-20T03:00:57,884 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing fc28de84807b58cb0d6ceaa73127debb, disabling compactions & flushes 2024-11-20T03:00:57,884 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,fff,1732071652347.fc28de84807b58cb0d6ceaa73127debb. 2024-11-20T03:00:57,884 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,fff,1732071652347.fc28de84807b58cb0d6ceaa73127debb. 2024-11-20T03:00:57,884 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,fff,1732071652347.fc28de84807b58cb0d6ceaa73127debb. after waiting 0 ms 2024-11-20T03:00:57,884 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,fff,1732071652347.fc28de84807b58cb0d6ceaa73127debb. 2024-11-20T03:00:57,887 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,rrr,1732071652347.a641a5878ac82d20c96fa3dff0720791. 2024-11-20T03:00:57,887 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for a641a5878ac82d20c96fa3dff0720791: Waiting for close lock at 1732071657877Running coprocessor pre-close hooks at 1732071657877Disabling compacts and flushes for region at 1732071657877Disabling writes for close at 1732071657878 (+1 ms)Writing region close event to WAL at 1732071657878Running coprocessor post-close hooks at 1732071657887 (+9 ms)Closed at 1732071657887 2024-11-20T03:00:57,888 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,rrr,1732071652347.a641a5878ac82d20c96fa3dff0720791. 2024-11-20T03:00:57,888 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 295cabe34d2d3dbccccb06cf3aa5bbf6, disabling compactions & flushes 2024-11-20T03:00:57,888 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,yyy,1732071652347.295cabe34d2d3dbccccb06cf3aa5bbf6. 2024-11-20T03:00:57,888 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,yyy,1732071652347.295cabe34d2d3dbccccb06cf3aa5bbf6. 2024-11-20T03:00:57,888 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,yyy,1732071652347.295cabe34d2d3dbccccb06cf3aa5bbf6. after waiting 0 ms 2024-11-20T03:00:57,888 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,yyy,1732071652347.295cabe34d2d3dbccccb06cf3aa5bbf6. 2024-11-20T03:00:57,890 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/default/test/fc28de84807b58cb0d6ceaa73127debb/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T03:00:57,892 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,fff,1732071652347.fc28de84807b58cb0d6ceaa73127debb. 2024-11-20T03:00:57,892 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for fc28de84807b58cb0d6ceaa73127debb: Waiting for close lock at 1732071657884Running coprocessor pre-close hooks at 1732071657884Disabling compacts and flushes for region at 1732071657884Disabling writes for close at 1732071657884Writing region close event to WAL at 1732071657885 (+1 ms)Running coprocessor post-close hooks at 1732071657892 (+7 ms)Closed at 1732071657892 2024-11-20T03:00:57,892 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,fff,1732071652347.fc28de84807b58cb0d6ceaa73127debb. 2024-11-20T03:00:57,892 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 8a818d7f8283ee7f2b5fae844b64a8ee, disabling compactions & flushes 2024-11-20T03:00:57,892 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ttt,1732071652347.8a818d7f8283ee7f2b5fae844b64a8ee. 2024-11-20T03:00:57,893 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ttt,1732071652347.8a818d7f8283ee7f2b5fae844b64a8ee. 2024-11-20T03:00:57,893 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ttt,1732071652347.8a818d7f8283ee7f2b5fae844b64a8ee. after waiting 0 ms 2024-11-20T03:00:57,893 INFO [RS_CLOSE_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/hbase/meta/1588230740/info/40d0cc0c57104bc6a92ac4d998695d40, entries=24, sequenceid=71, filesize=8 K 2024-11-20T03:00:57,893 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ttt,1732071652347.8a818d7f8283ee7f2b5fae844b64a8ee. 2024-11-20T03:00:57,893 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/default/test/295cabe34d2d3dbccccb06cf3aa5bbf6/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T03:00:57,894 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,yyy,1732071652347.295cabe34d2d3dbccccb06cf3aa5bbf6. 2024-11-20T03:00:57,894 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 295cabe34d2d3dbccccb06cf3aa5bbf6: Waiting for close lock at 1732071657888Running coprocessor pre-close hooks at 1732071657888Disabling compacts and flushes for region at 1732071657888Disabling writes for close at 1732071657888Writing region close event to WAL at 1732071657889 (+1 ms)Running coprocessor post-close hooks at 1732071657894 (+5 ms)Closed at 1732071657894 2024-11-20T03:00:57,894 INFO [RS_CLOSE_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~2.65 KB/2717, heapSize ~4.67 KB/4784, currentSize=0 B/0 for 1588230740 in 59ms, sequenceid=71, compaction requested=false 2024-11-20T03:00:57,894 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,yyy,1732071652347.295cabe34d2d3dbccccb06cf3aa5bbf6. 2024-11-20T03:00:57,894 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing ccd7aea6ec14848f8c6730c88ce1c780, disabling compactions & flushes 2024-11-20T03:00:57,894 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ccc,1732071652347.ccd7aea6ec14848f8c6730c88ce1c780. 2024-11-20T03:00:57,894 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ccc,1732071652347.ccd7aea6ec14848f8c6730c88ce1c780. 2024-11-20T03:00:57,895 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ccc,1732071652347.ccd7aea6ec14848f8c6730c88ce1c780. after waiting 1 ms 2024-11-20T03:00:57,895 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ccc,1732071652347.ccd7aea6ec14848f8c6730c88ce1c780. 2024-11-20T03:00:57,897 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/default/test/8a818d7f8283ee7f2b5fae844b64a8ee/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T03:00:57,898 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ttt,1732071652347.8a818d7f8283ee7f2b5fae844b64a8ee. 2024-11-20T03:00:57,898 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 8a818d7f8283ee7f2b5fae844b64a8ee: Waiting for close lock at 1732071657892Running coprocessor pre-close hooks at 1732071657892Disabling compacts and flushes for region at 1732071657892Disabling writes for close at 1732071657893 (+1 ms)Writing region close event to WAL at 1732071657893Running coprocessor post-close hooks at 1732071657898 (+5 ms)Closed at 1732071657898 2024-11-20T03:00:57,899 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ttt,1732071652347.8a818d7f8283ee7f2b5fae844b64a8ee. 2024-11-20T03:00:57,899 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing ef70776250de5f4e8ad7bd24027abe63, disabling compactions & flushes 2024-11-20T03:00:57,899 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ppp,1732071652347.ef70776250de5f4e8ad7bd24027abe63. 2024-11-20T03:00:57,899 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ppp,1732071652347.ef70776250de5f4e8ad7bd24027abe63. 2024-11-20T03:00:57,899 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ppp,1732071652347.ef70776250de5f4e8ad7bd24027abe63. after waiting 0 ms 2024-11-20T03:00:57,899 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ppp,1732071652347.ef70776250de5f4e8ad7bd24027abe63. 2024-11-20T03:00:57,901 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/default/test/ccd7aea6ec14848f8c6730c88ce1c780/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T03:00:57,901 DEBUG [RS_CLOSE_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/hbase/meta/1588230740/recovered.edits/74.seqid, newMaxSeqId=74, maxSeqId=60 2024-11-20T03:00:57,902 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ccc,1732071652347.ccd7aea6ec14848f8c6730c88ce1c780. 2024-11-20T03:00:57,902 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for ccd7aea6ec14848f8c6730c88ce1c780: Waiting for close lock at 1732071657894Running coprocessor pre-close hooks at 1732071657894Disabling compacts and flushes for region at 1732071657894Disabling writes for close at 1732071657895 (+1 ms)Writing region close event to WAL at 1732071657895Running coprocessor post-close hooks at 1732071657902 (+7 ms)Closed at 1732071657902 2024-11-20T03:00:57,902 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ccc,1732071652347.ccd7aea6ec14848f8c6730c88ce1c780. 2024-11-20T03:00:57,902 DEBUG [RS_CLOSE_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-20T03:00:57,902 INFO [RS_CLOSE_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-20T03:00:57,902 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing f928be6f5d7a3631384ebd87c8f80996, disabling compactions & flushes 2024-11-20T03:00:57,902 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,iii,1732071652347.f928be6f5d7a3631384ebd87c8f80996. 2024-11-20T03:00:57,902 DEBUG [RS_CLOSE_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732071657835Running coprocessor pre-close hooks at 1732071657835Disabling compacts and flushes for region at 1732071657835Disabling writes for close at 1732071657835Obtaining lock to block concurrent updates at 1732071657835Preparing flush snapshotting stores in 1588230740 at 1732071657835Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=2717, getHeapSize=5504, getOffHeapSize=0, getCellsCount=24 at 1732071657843 (+8 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732071657844 (+1 ms)Flushing 1588230740/info: creating writer at 1732071657845 (+1 ms)Flushing 1588230740/info: appending metadata at 1732071657850 (+5 ms)Flushing 1588230740/info: closing flushed file at 1732071657850Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@26bffe0d: reopening flushed file at 1732071657880 (+30 ms)Finished flush of dataSize ~2.65 KB/2717, heapSize ~4.67 KB/4784, currentSize=0 B/0 for 1588230740 in 59ms, sequenceid=71, compaction requested=false at 1732071657894 (+14 ms)Writing region close event to WAL at 1732071657896 (+2 ms)Running coprocessor post-close hooks at 1732071657902 (+6 ms)Closed at 1732071657902 2024-11-20T03:00:57,902 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,iii,1732071652347.f928be6f5d7a3631384ebd87c8f80996. 2024-11-20T03:00:57,902 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,iii,1732071652347.f928be6f5d7a3631384ebd87c8f80996. after waiting 0 ms 2024-11-20T03:00:57,902 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,iii,1732071652347.f928be6f5d7a3631384ebd87c8f80996. 2024-11-20T03:00:57,902 DEBUG [RS_CLOSE_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-20T03:00:57,905 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/default/test/ef70776250de5f4e8ad7bd24027abe63/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T03:00:57,905 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ppp,1732071652347.ef70776250de5f4e8ad7bd24027abe63. 2024-11-20T03:00:57,905 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for ef70776250de5f4e8ad7bd24027abe63: Waiting for close lock at 1732071657899Running coprocessor pre-close hooks at 1732071657899Disabling compacts and flushes for region at 1732071657899Disabling writes for close at 1732071657899Writing region close event to WAL at 1732071657900 (+1 ms)Running coprocessor post-close hooks at 1732071657905 (+5 ms)Closed at 1732071657905 2024-11-20T03:00:57,905 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ppp,1732071652347.ef70776250de5f4e8ad7bd24027abe63. 2024-11-20T03:00:57,906 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 38fa6330e381ad6ba7391bf9050c289e, disabling compactions & flushes 2024-11-20T03:00:57,906 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,mmm,1732071652347.38fa6330e381ad6ba7391bf9050c289e. 2024-11-20T03:00:57,906 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,mmm,1732071652347.38fa6330e381ad6ba7391bf9050c289e. 2024-11-20T03:00:57,906 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,mmm,1732071652347.38fa6330e381ad6ba7391bf9050c289e. after waiting 0 ms 2024-11-20T03:00:57,906 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,mmm,1732071652347.38fa6330e381ad6ba7391bf9050c289e. 2024-11-20T03:00:57,908 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/default/test/f928be6f5d7a3631384ebd87c8f80996/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T03:00:57,908 INFO [regionserver/1c82b83631a2:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T03:00:57,908 INFO [regionserver/1c82b83631a2:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T03:00:57,909 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,iii,1732071652347.f928be6f5d7a3631384ebd87c8f80996. 2024-11-20T03:00:57,909 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for f928be6f5d7a3631384ebd87c8f80996: Waiting for close lock at 1732071657902Running coprocessor pre-close hooks at 1732071657902Disabling compacts and flushes for region at 1732071657902Disabling writes for close at 1732071657902Writing region close event to WAL at 1732071657903 (+1 ms)Running coprocessor post-close hooks at 1732071657908 (+5 ms)Closed at 1732071657909 (+1 ms) 2024-11-20T03:00:57,909 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,iii,1732071652347.f928be6f5d7a3631384ebd87c8f80996. 2024-11-20T03:00:57,909 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 4327d986b4dd716142908b96606b0ad7, disabling compactions & flushes 2024-11-20T03:00:57,909 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ooo,1732071652347.4327d986b4dd716142908b96606b0ad7. 2024-11-20T03:00:57,909 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ooo,1732071652347.4327d986b4dd716142908b96606b0ad7. 2024-11-20T03:00:57,909 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ooo,1732071652347.4327d986b4dd716142908b96606b0ad7. after waiting 0 ms 2024-11-20T03:00:57,909 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ooo,1732071652347.4327d986b4dd716142908b96606b0ad7. 2024-11-20T03:00:57,911 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/default/test/38fa6330e381ad6ba7391bf9050c289e/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T03:00:57,912 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,mmm,1732071652347.38fa6330e381ad6ba7391bf9050c289e. 2024-11-20T03:00:57,912 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 38fa6330e381ad6ba7391bf9050c289e: Waiting for close lock at 1732071657906Running coprocessor pre-close hooks at 1732071657906Disabling compacts and flushes for region at 1732071657906Disabling writes for close at 1732071657906Writing region close event to WAL at 1732071657906Running coprocessor post-close hooks at 1732071657912 (+6 ms)Closed at 1732071657912 2024-11-20T03:00:57,912 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,mmm,1732071652347.38fa6330e381ad6ba7391bf9050c289e. 2024-11-20T03:00:57,912 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing e8e6d28d8e1806fa7d958327b87df211, disabling compactions & flushes 2024-11-20T03:00:57,912 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,xxx,1732071652347.e8e6d28d8e1806fa7d958327b87df211. 2024-11-20T03:00:57,912 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,xxx,1732071652347.e8e6d28d8e1806fa7d958327b87df211. 2024-11-20T03:00:57,912 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,xxx,1732071652347.e8e6d28d8e1806fa7d958327b87df211. after waiting 0 ms 2024-11-20T03:00:57,912 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,xxx,1732071652347.e8e6d28d8e1806fa7d958327b87df211. 2024-11-20T03:00:57,914 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/default/test/4327d986b4dd716142908b96606b0ad7/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T03:00:57,915 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ooo,1732071652347.4327d986b4dd716142908b96606b0ad7. 2024-11-20T03:00:57,915 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 4327d986b4dd716142908b96606b0ad7: Waiting for close lock at 1732071657909Running coprocessor pre-close hooks at 1732071657909Disabling compacts and flushes for region at 1732071657909Disabling writes for close at 1732071657909Writing region close event to WAL at 1732071657910 (+1 ms)Running coprocessor post-close hooks at 1732071657915 (+5 ms)Closed at 1732071657915 2024-11-20T03:00:57,915 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ooo,1732071652347.4327d986b4dd716142908b96606b0ad7. 2024-11-20T03:00:57,915 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 550bbfe722915166ce74fec36828c508, disabling compactions & flushes 2024-11-20T03:00:57,915 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,sss,1732071652347.550bbfe722915166ce74fec36828c508. 2024-11-20T03:00:57,915 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,sss,1732071652347.550bbfe722915166ce74fec36828c508. 2024-11-20T03:00:57,916 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,sss,1732071652347.550bbfe722915166ce74fec36828c508. after waiting 0 ms 2024-11-20T03:00:57,916 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,sss,1732071652347.550bbfe722915166ce74fec36828c508. 2024-11-20T03:00:57,917 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/default/test/e8e6d28d8e1806fa7d958327b87df211/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T03:00:57,918 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,xxx,1732071652347.e8e6d28d8e1806fa7d958327b87df211. 2024-11-20T03:00:57,918 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for e8e6d28d8e1806fa7d958327b87df211: Waiting for close lock at 1732071657912Running coprocessor pre-close hooks at 1732071657912Disabling compacts and flushes for region at 1732071657912Disabling writes for close at 1732071657912Writing region close event to WAL at 1732071657913 (+1 ms)Running coprocessor post-close hooks at 1732071657918 (+5 ms)Closed at 1732071657918 2024-11-20T03:00:57,918 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,xxx,1732071652347.e8e6d28d8e1806fa7d958327b87df211. 2024-11-20T03:00:57,918 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing fbf7ef74de7c98db6b37604b08e67282, disabling compactions & flushes 2024-11-20T03:00:57,918 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,nnn,1732071652347.fbf7ef74de7c98db6b37604b08e67282. 2024-11-20T03:00:57,918 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,nnn,1732071652347.fbf7ef74de7c98db6b37604b08e67282. 2024-11-20T03:00:57,919 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,nnn,1732071652347.fbf7ef74de7c98db6b37604b08e67282. after waiting 0 ms 2024-11-20T03:00:57,919 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,nnn,1732071652347.fbf7ef74de7c98db6b37604b08e67282. 2024-11-20T03:00:57,920 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/default/test/550bbfe722915166ce74fec36828c508/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T03:00:57,921 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,sss,1732071652347.550bbfe722915166ce74fec36828c508. 2024-11-20T03:00:57,921 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 550bbfe722915166ce74fec36828c508: Waiting for close lock at 1732071657915Running coprocessor pre-close hooks at 1732071657915Disabling compacts and flushes for region at 1732071657915Disabling writes for close at 1732071657916 (+1 ms)Writing region close event to WAL at 1732071657916Running coprocessor post-close hooks at 1732071657921 (+5 ms)Closed at 1732071657921 2024-11-20T03:00:57,921 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,sss,1732071652347.550bbfe722915166ce74fec36828c508. 2024-11-20T03:00:57,922 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 2fe87cdaf67461ff5c508d851e418c66, disabling compactions & flushes 2024-11-20T03:00:57,922 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,eee,1732071652347.2fe87cdaf67461ff5c508d851e418c66. 2024-11-20T03:00:57,922 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,eee,1732071652347.2fe87cdaf67461ff5c508d851e418c66. 2024-11-20T03:00:57,922 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,eee,1732071652347.2fe87cdaf67461ff5c508d851e418c66. after waiting 0 ms 2024-11-20T03:00:57,922 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,eee,1732071652347.2fe87cdaf67461ff5c508d851e418c66. 2024-11-20T03:00:57,923 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/default/test/fbf7ef74de7c98db6b37604b08e67282/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T03:00:57,924 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,nnn,1732071652347.fbf7ef74de7c98db6b37604b08e67282. 2024-11-20T03:00:57,924 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for fbf7ef74de7c98db6b37604b08e67282: Waiting for close lock at 1732071657918Running coprocessor pre-close hooks at 1732071657918Disabling compacts and flushes for region at 1732071657918Disabling writes for close at 1732071657919 (+1 ms)Writing region close event to WAL at 1732071657919Running coprocessor post-close hooks at 1732071657924 (+5 ms)Closed at 1732071657924 2024-11-20T03:00:57,924 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,nnn,1732071652347.fbf7ef74de7c98db6b37604b08e67282. 2024-11-20T03:00:57,924 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 3c5e70743a336dce38e44ee0f8059378, disabling compactions & flushes 2024-11-20T03:00:57,924 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ggg,1732071652347.3c5e70743a336dce38e44ee0f8059378. 2024-11-20T03:00:57,925 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ggg,1732071652347.3c5e70743a336dce38e44ee0f8059378. 2024-11-20T03:00:57,925 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ggg,1732071652347.3c5e70743a336dce38e44ee0f8059378. after waiting 0 ms 2024-11-20T03:00:57,925 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ggg,1732071652347.3c5e70743a336dce38e44ee0f8059378. 2024-11-20T03:00:57,926 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/default/test/2fe87cdaf67461ff5c508d851e418c66/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T03:00:57,927 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,eee,1732071652347.2fe87cdaf67461ff5c508d851e418c66. 2024-11-20T03:00:57,927 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 2fe87cdaf67461ff5c508d851e418c66: Waiting for close lock at 1732071657922Running coprocessor pre-close hooks at 1732071657922Disabling compacts and flushes for region at 1732071657922Disabling writes for close at 1732071657922Writing region close event to WAL at 1732071657922Running coprocessor post-close hooks at 1732071657927 (+5 ms)Closed at 1732071657927 2024-11-20T03:00:57,927 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,eee,1732071652347.2fe87cdaf67461ff5c508d851e418c66. 2024-11-20T03:00:57,927 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 45e8fc1ea8df34f038aeb00318494097, disabling compactions & flushes 2024-11-20T03:00:57,927 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,vvv,1732071652347.45e8fc1ea8df34f038aeb00318494097. 2024-11-20T03:00:57,928 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,vvv,1732071652347.45e8fc1ea8df34f038aeb00318494097. 2024-11-20T03:00:57,928 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,vvv,1732071652347.45e8fc1ea8df34f038aeb00318494097. after waiting 0 ms 2024-11-20T03:00:57,928 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,vvv,1732071652347.45e8fc1ea8df34f038aeb00318494097. 2024-11-20T03:00:57,929 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/default/test/3c5e70743a336dce38e44ee0f8059378/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T03:00:57,930 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ggg,1732071652347.3c5e70743a336dce38e44ee0f8059378. 2024-11-20T03:00:57,930 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 3c5e70743a336dce38e44ee0f8059378: Waiting for close lock at 1732071657924Running coprocessor pre-close hooks at 1732071657924Disabling compacts and flushes for region at 1732071657924Disabling writes for close at 1732071657925 (+1 ms)Writing region close event to WAL at 1732071657925Running coprocessor post-close hooks at 1732071657930 (+5 ms)Closed at 1732071657930 2024-11-20T03:00:57,930 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ggg,1732071652347.3c5e70743a336dce38e44ee0f8059378. 2024-11-20T03:00:57,932 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/data/default/test/45e8fc1ea8df34f038aeb00318494097/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T03:00:57,933 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,vvv,1732071652347.45e8fc1ea8df34f038aeb00318494097. 2024-11-20T03:00:57,933 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 45e8fc1ea8df34f038aeb00318494097: Waiting for close lock at 1732071657927Running coprocessor pre-close hooks at 1732071657927Disabling compacts and flushes for region at 1732071657927Disabling writes for close at 1732071657928 (+1 ms)Writing region close event to WAL at 1732071657928Running coprocessor post-close hooks at 1732071657932 (+4 ms)Closed at 1732071657932 2024-11-20T03:00:57,933 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,vvv,1732071652347.45e8fc1ea8df34f038aeb00318494097. 2024-11-20T03:00:58,026 INFO [RS:0;1c82b83631a2:36133 {}] regionserver.HRegionServer(976): stopping server 1c82b83631a2,36133,1732071649570; all regions closed. 2024-11-20T03:00:58,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33951 is added to blk_1073741835_1011 (size=7074) 2024-11-20T03:00:58,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42923 is added to blk_1073741835_1011 (size=7074) 2024-11-20T03:00:58,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38873 is added to blk_1073741835_1011 (size=7074) 2024-11-20T03:00:58,032 DEBUG [RS:0;1c82b83631a2:36133 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/oldWALs 2024-11-20T03:00:58,032 INFO [RS:0;1c82b83631a2:36133 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 1c82b83631a2%2C36133%2C1732071649570:(num 1732071651367) 2024-11-20T03:00:58,032 DEBUG [RS:0;1c82b83631a2:36133 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T03:00:58,032 INFO [RS:0;1c82b83631a2:36133 {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T03:00:58,032 INFO [RS:0;1c82b83631a2:36133 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-20T03:00:58,032 INFO [RS:0;1c82b83631a2:36133 {}] hbase.ChoreService(370): Chore service for: regionserver/1c82b83631a2:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-20T03:00:58,033 INFO [RS:0;1c82b83631a2:36133 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-20T03:00:58,033 INFO [RS:0;1c82b83631a2:36133 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-20T03:00:58,033 INFO [regionserver/1c82b83631a2:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-20T03:00:58,033 INFO [RS:0;1c82b83631a2:36133 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-20T03:00:58,033 INFO [RS:0;1c82b83631a2:36133 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-20T03:00:58,033 INFO [RS:0;1c82b83631a2:36133 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36133 2024-11-20T03:00:58,035 INFO [RS:1;1c82b83631a2:42975 {}] regionserver.HRegionServer(976): stopping server 1c82b83631a2,42975,1732071649704; all regions closed. 2024-11-20T03:00:58,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42923 is added to blk_1073741865_1041 (size=4201) 2024-11-20T03:00:58,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33951 is added to blk_1073741865_1041 (size=4201) 2024-11-20T03:00:58,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38873 is added to blk_1073741865_1041 (size=4201) 2024-11-20T03:00:58,038 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/WALs/1c82b83631a2,42975,1732071649704/1c82b83631a2%2C42975%2C1732071649704.meta.1732071655501.meta not finished, retry = 0 2024-11-20T03:00:58,056 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36133-0x10157797d910001, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/1c82b83631a2,36133,1732071649570 2024-11-20T03:00:58,056 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39617-0x10157797d910000, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-20T03:00:58,056 INFO [RS:0;1c82b83631a2:36133 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-20T03:00:58,065 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [1c82b83631a2,36133,1732071649570] 2024-11-20T03:00:58,073 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/1c82b83631a2,36133,1732071649570 already deleted, retry=false 2024-11-20T03:00:58,073 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 1c82b83631a2,36133,1732071649570 expired; onlineServers=1 2024-11-20T03:00:58,144 DEBUG [RS:1;1c82b83631a2:42975 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/oldWALs 2024-11-20T03:00:58,144 INFO [RS:1;1c82b83631a2:42975 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 1c82b83631a2%2C42975%2C1732071649704.meta:.meta(num 1732071655501) 2024-11-20T03:00:58,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38873 is added to blk_1073741833_1009 (size=7064) 2024-11-20T03:00:58,148 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/WALs/1c82b83631a2,42975,1732071649704/1c82b83631a2%2C42975%2C1732071649704.1732071651367 not finished, retry = 0 2024-11-20T03:00:58,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33951 is added to blk_1073741833_1009 (size=7064) 2024-11-20T03:00:58,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42923 is added to blk_1073741833_1009 (size=7064) 2024-11-20T03:00:58,165 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36133-0x10157797d910001, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T03:00:58,165 INFO [RS:0;1c82b83631a2:36133 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-20T03:00:58,165 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36133-0x10157797d910001, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T03:00:58,165 INFO [RS:0;1c82b83631a2:36133 {}] regionserver.HRegionServer(1031): Exiting; stopping=1c82b83631a2,36133,1732071649570; zookeeper connection closed. 2024-11-20T03:00:58,165 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@6e4fd9bc {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@6e4fd9bc 2024-11-20T03:00:58,195 INFO [regionserver/1c82b83631a2:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-20T03:00:58,195 INFO [regionserver/1c82b83631a2:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-20T03:00:58,256 DEBUG [RS:1;1c82b83631a2:42975 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/oldWALs 2024-11-20T03:00:58,256 INFO [RS:1;1c82b83631a2:42975 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 1c82b83631a2%2C42975%2C1732071649704:(num 1732071651367) 2024-11-20T03:00:58,256 DEBUG [RS:1;1c82b83631a2:42975 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T03:00:58,256 INFO [RS:1;1c82b83631a2:42975 {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T03:00:58,256 INFO [RS:1;1c82b83631a2:42975 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-20T03:00:58,257 INFO [RS:1;1c82b83631a2:42975 {}] hbase.ChoreService(370): Chore service for: regionserver/1c82b83631a2:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-20T03:00:58,257 INFO [RS:1;1c82b83631a2:42975 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-20T03:00:58,257 INFO [regionserver/1c82b83631a2:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-20T03:00:58,258 INFO [RS:1;1c82b83631a2:42975 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42975 2024-11-20T03:00:58,279 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39617-0x10157797d910000, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-20T03:00:58,279 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42975-0x10157797d910002, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/1c82b83631a2,42975,1732071649704 2024-11-20T03:00:58,279 INFO [RS:1;1c82b83631a2:42975 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-20T03:00:58,289 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [1c82b83631a2,42975,1732071649704] 2024-11-20T03:00:58,298 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/1c82b83631a2,42975,1732071649704 already deleted, retry=false 2024-11-20T03:00:58,298 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 1c82b83631a2,42975,1732071649704 expired; onlineServers=0 2024-11-20T03:00:58,298 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '1c82b83631a2,39617,1732071648799' ***** 2024-11-20T03:00:58,298 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-20T03:00:58,298 INFO [M:0;1c82b83631a2:39617 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-20T03:00:58,298 INFO [M:0;1c82b83631a2:39617 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-20T03:00:58,298 DEBUG [M:0;1c82b83631a2:39617 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-20T03:00:58,298 DEBUG [M:0;1c82b83631a2:39617 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-20T03:00:58,298 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-20T03:00:58,298 DEBUG [master/1c82b83631a2:0:becomeActiveMaster-HFileCleaner.small.0-1732071650978 {}] cleaner.HFileCleaner(306): Exit Thread[master/1c82b83631a2:0:becomeActiveMaster-HFileCleaner.small.0-1732071650978,5,FailOnTimeoutGroup] 2024-11-20T03:00:58,298 DEBUG [master/1c82b83631a2:0:becomeActiveMaster-HFileCleaner.large.0-1732071650964 {}] cleaner.HFileCleaner(306): Exit Thread[master/1c82b83631a2:0:becomeActiveMaster-HFileCleaner.large.0-1732071650964,5,FailOnTimeoutGroup] 2024-11-20T03:00:58,299 INFO [M:0;1c82b83631a2:39617 {}] hbase.ChoreService(370): Chore service for: master/1c82b83631a2:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-20T03:00:58,299 INFO [M:0;1c82b83631a2:39617 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-20T03:00:58,299 DEBUG [M:0;1c82b83631a2:39617 {}] master.HMaster(1795): Stopping service threads 2024-11-20T03:00:58,299 INFO [M:0;1c82b83631a2:39617 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-20T03:00:58,300 INFO [M:0;1c82b83631a2:39617 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T03:00:58,300 INFO [M:0;1c82b83631a2:39617 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T03:00:58,300 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-20T03:00:58,306 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39617-0x10157797d910000, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-20T03:00:58,306 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39617-0x10157797d910000, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T03:00:58,307 DEBUG [M:0;1c82b83631a2:39617 {}] zookeeper.ZKUtil(347): master:39617-0x10157797d910000, quorum=127.0.0.1:52318, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-20T03:00:58,307 WARN [M:0;1c82b83631a2:39617 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-20T03:00:58,308 INFO [M:0;1c82b83631a2:39617 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/.lastflushedseqids 2024-11-20T03:00:58,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42923 is added to blk_1073741870_1046 (size=1) 2024-11-20T03:00:58,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33951 is added to blk_1073741870_1046 (size=1) 2024-11-20T03:00:58,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38873 is added to blk_1073741870_1046 (size=1) 2024-11-20T03:00:58,319 INFO [M:0;1c82b83631a2:39617 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T03:00:58,320 INFO [M:0;1c82b83631a2:39617 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-20T03:00:58,320 DEBUG [M:0;1c82b83631a2:39617 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-20T03:00:58,320 INFO [M:0;1c82b83631a2:39617 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T03:00:58,320 DEBUG [M:0;1c82b83631a2:39617 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T03:00:58,320 DEBUG [M:0;1c82b83631a2:39617 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-20T03:00:58,320 DEBUG [M:0;1c82b83631a2:39617 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T03:00:58,320 INFO [M:0;1c82b83631a2:39617 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=250.04 KB heapSize=303.89 KB 2024-11-20T03:00:58,335 DEBUG [M:0;1c82b83631a2:39617 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/16b85d08aa7744f29635cd34ed5aa97c is 82, key is hbase:meta,,1/info:regioninfo/1732071655882/Put/seqid=0 2024-11-20T03:00:58,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42923 is added to blk_1073741871_1047 (size=6071) 2024-11-20T03:00:58,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38873 is added to blk_1073741871_1047 (size=6071) 2024-11-20T03:00:58,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33951 is added to blk_1073741871_1047 (size=6071) 2024-11-20T03:00:58,343 INFO [M:0;1c82b83631a2:39617 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=575 (bloomFilter=true), to=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/16b85d08aa7744f29635cd34ed5aa97c 2024-11-20T03:00:58,372 DEBUG [M:0;1c82b83631a2:39617 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/2d6f415ca93a409ab7050daf15bf89ca is 2987, key is \x00\x00\x00\x00\x00\x00\x009/proc:d/1732071657786/Put/seqid=0 2024-11-20T03:00:58,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33951 is added to blk_1073741872_1048 (size=12797) 2024-11-20T03:00:58,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38873 is added to blk_1073741872_1048 (size=12797) 2024-11-20T03:00:58,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42923 is added to blk_1073741872_1048 (size=12797) 2024-11-20T03:00:58,390 INFO [M:0;1c82b83631a2:39617 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=248.76 KB at sequenceid=575 (bloomFilter=true), to=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/2d6f415ca93a409ab7050daf15bf89ca 2024-11-20T03:00:58,390 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42975-0x10157797d910002, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T03:00:58,390 INFO [RS:1;1c82b83631a2:42975 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-20T03:00:58,390 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42975-0x10157797d910002, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T03:00:58,390 INFO [RS:1;1c82b83631a2:42975 {}] regionserver.HRegionServer(1031): Exiting; stopping=1c82b83631a2,42975,1732071649704; zookeeper connection closed. 2024-11-20T03:00:58,390 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@7f9dd964 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@7f9dd964 2024-11-20T03:00:58,391 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-20T03:00:58,414 DEBUG [M:0;1c82b83631a2:39617 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e2023c8996734f9caeaa5b6db622d0bd is 69, key is 1c82b83631a2,36133,1732071649570/rs:state/1732071651068/Put/seqid=0 2024-11-20T03:00:58,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33951 is added to blk_1073741873_1049 (size=5442) 2024-11-20T03:00:58,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42923 is added to blk_1073741873_1049 (size=5442) 2024-11-20T03:00:58,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38873 is added to blk_1073741873_1049 (size=5442) 2024-11-20T03:00:58,422 INFO [M:0;1c82b83631a2:39617 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=575 (bloomFilter=true), to=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e2023c8996734f9caeaa5b6db622d0bd 2024-11-20T03:00:58,428 INFO [M:0;1c82b83631a2:39617 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for e2023c8996734f9caeaa5b6db622d0bd 2024-11-20T03:00:58,451 DEBUG [M:0;1c82b83631a2:39617 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/9a46760d7ae54e91905264076de69395 is 52, key is load_balancer_on/state:d/1732071654686/Put/seqid=0 2024-11-20T03:00:58,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33951 is added to blk_1073741874_1050 (size=5058) 2024-11-20T03:00:58,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38873 is added to blk_1073741874_1050 (size=5058) 2024-11-20T03:00:58,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42923 is added to blk_1073741874_1050 (size=5058) 2024-11-20T03:00:58,459 INFO [M:0;1c82b83631a2:39617 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=575 (bloomFilter=true), to=hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/9a46760d7ae54e91905264076de69395 2024-11-20T03:00:58,469 DEBUG [M:0;1c82b83631a2:39617 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/16b85d08aa7744f29635cd34ed5aa97c as hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/16b85d08aa7744f29635cd34ed5aa97c 2024-11-20T03:00:58,476 INFO [M:0;1c82b83631a2:39617 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/16b85d08aa7744f29635cd34ed5aa97c, entries=14, sequenceid=575, filesize=5.9 K 2024-11-20T03:00:58,477 DEBUG [M:0;1c82b83631a2:39617 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/2d6f415ca93a409ab7050daf15bf89ca as hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/2d6f415ca93a409ab7050daf15bf89ca 2024-11-20T03:00:58,485 INFO [M:0;1c82b83631a2:39617 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/2d6f415ca93a409ab7050daf15bf89ca, entries=79, sequenceid=575, filesize=12.5 K 2024-11-20T03:00:58,486 DEBUG [M:0;1c82b83631a2:39617 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e2023c8996734f9caeaa5b6db622d0bd as hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/e2023c8996734f9caeaa5b6db622d0bd 2024-11-20T03:00:58,493 INFO [M:0;1c82b83631a2:39617 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for e2023c8996734f9caeaa5b6db622d0bd 2024-11-20T03:00:58,493 INFO [M:0;1c82b83631a2:39617 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/e2023c8996734f9caeaa5b6db622d0bd, entries=3, sequenceid=575, filesize=5.3 K 2024-11-20T03:00:58,495 DEBUG [M:0;1c82b83631a2:39617 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/9a46760d7ae54e91905264076de69395 as hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/9a46760d7ae54e91905264076de69395 2024-11-20T03:00:58,502 INFO [M:0;1c82b83631a2:39617 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38107/user/jenkins/test-data/dc42770c-7ce7-faad-d380-fbe7733d4766/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/9a46760d7ae54e91905264076de69395, entries=1, sequenceid=575, filesize=4.9 K 2024-11-20T03:00:58,503 INFO [M:0;1c82b83631a2:39617 {}] regionserver.HRegion(3140): Finished flush of dataSize ~250.04 KB/256039, heapSize ~303.83 KB/311120, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 183ms, sequenceid=575, compaction requested=false 2024-11-20T03:00:58,505 INFO [M:0;1c82b83631a2:39617 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T03:00:58,505 DEBUG [M:0;1c82b83631a2:39617 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732071658320Disabling compacts and flushes for region at 1732071658320Disabling writes for close at 1732071658320Obtaining lock to block concurrent updates at 1732071658320Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732071658320Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=256039, getHeapSize=311120, getOffHeapSize=0, getCellsCount=714 at 1732071658320Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732071658321 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732071658321Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732071658335 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732071658335Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732071658350 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732071658371 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732071658371Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732071658398 (+27 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732071658413 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732071658413Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1732071658428 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1732071658450 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1732071658450Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@58f5725d: reopening flushed file at 1732071658468 (+18 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5aee4119: reopening flushed file at 1732071658476 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1cf5925f: reopening flushed file at 1732071658485 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@678508ca: reopening flushed file at 1732071658494 (+9 ms)Finished flush of dataSize ~250.04 KB/256039, heapSize ~303.83 KB/311120, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 183ms, sequenceid=575, compaction requested=false at 1732071658503 (+9 ms)Writing region close event to WAL at 1732071658505 (+2 ms)Closed at 1732071658505 2024-11-20T03:00:58,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38873 is added to blk_1073741830_1006 (size=294423) 2024-11-20T03:00:58,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33951 is added to blk_1073741830_1006 (size=294423) 2024-11-20T03:00:58,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42923 is added to blk_1073741830_1006 (size=294423) 2024-11-20T03:00:58,509 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-20T03:00:58,509 INFO [M:0;1c82b83631a2:39617 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-20T03:00:58,509 INFO [M:0;1c82b83631a2:39617 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39617 2024-11-20T03:00:58,509 INFO [M:0;1c82b83631a2:39617 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-20T03:00:58,640 INFO [M:0;1c82b83631a2:39617 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-20T03:00:58,640 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39617-0x10157797d910000, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T03:00:58,640 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39617-0x10157797d910000, quorum=127.0.0.1:52318, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T03:00:58,644 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@67ddfd25{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-20T03:00:58,646 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@30e38263{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-20T03:00:58,646 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-20T03:00:58,646 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@52cf89e6{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-20T03:00:58,647 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@11ebaa12{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cc48e81-f8f9-9439-aadc-f08e2410e401/hadoop.log.dir/,STOPPED} 2024-11-20T03:00:58,649 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-20T03:00:58,649 WARN [BP-1182860866-172.17.0.2-1732071644397 heartbeating to localhost/127.0.0.1:38107 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-20T03:00:58,649 WARN [BP-1182860866-172.17.0.2-1732071644397 heartbeating to localhost/127.0.0.1:38107 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1182860866-172.17.0.2-1732071644397 (Datanode Uuid e8cd692b-1714-45ca-b85e-b6498b320236) service to localhost/127.0.0.1:38107 2024-11-20T03:00:58,649 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-20T03:00:58,650 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cc48e81-f8f9-9439-aadc-f08e2410e401/cluster_a1accffb-3df0-3830-8589-ade335756e45/data/data5/current/BP-1182860866-172.17.0.2-1732071644397 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-20T03:00:58,650 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cc48e81-f8f9-9439-aadc-f08e2410e401/cluster_a1accffb-3df0-3830-8589-ade335756e45/data/data6/current/BP-1182860866-172.17.0.2-1732071644397 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-20T03:00:58,650 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-20T03:00:58,652 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7cfccf27{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-20T03:00:58,653 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6c41bbb7{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-20T03:00:58,653 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-20T03:00:58,653 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@12f16529{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-20T03:00:58,653 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@34a3e1ff{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cc48e81-f8f9-9439-aadc-f08e2410e401/hadoop.log.dir/,STOPPED} 2024-11-20T03:00:58,655 WARN [BP-1182860866-172.17.0.2-1732071644397 heartbeating to localhost/127.0.0.1:38107 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-20T03:00:58,655 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-20T03:00:58,655 WARN [BP-1182860866-172.17.0.2-1732071644397 heartbeating to localhost/127.0.0.1:38107 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1182860866-172.17.0.2-1732071644397 (Datanode Uuid 5735bcd3-4753-44a1-aeed-94e7ab4d5722) service to localhost/127.0.0.1:38107 2024-11-20T03:00:58,655 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-20T03:00:58,655 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cc48e81-f8f9-9439-aadc-f08e2410e401/cluster_a1accffb-3df0-3830-8589-ade335756e45/data/data3/current/BP-1182860866-172.17.0.2-1732071644397 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-20T03:00:58,656 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cc48e81-f8f9-9439-aadc-f08e2410e401/cluster_a1accffb-3df0-3830-8589-ade335756e45/data/data4/current/BP-1182860866-172.17.0.2-1732071644397 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-20T03:00:58,656 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-20T03:00:58,658 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2226d6a6{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-20T03:00:58,658 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@1da1f773{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-20T03:00:58,658 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-20T03:00:58,658 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1c5d2af1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-20T03:00:58,658 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@739a104e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cc48e81-f8f9-9439-aadc-f08e2410e401/hadoop.log.dir/,STOPPED} 2024-11-20T03:00:58,659 WARN [BP-1182860866-172.17.0.2-1732071644397 heartbeating to localhost/127.0.0.1:38107 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-20T03:00:58,659 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-20T03:00:58,660 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-20T03:00:58,660 WARN [BP-1182860866-172.17.0.2-1732071644397 heartbeating to localhost/127.0.0.1:38107 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1182860866-172.17.0.2-1732071644397 (Datanode Uuid 500d6ffd-8429-4309-bae3-992a8c7dc324) service to localhost/127.0.0.1:38107 2024-11-20T03:00:58,660 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cc48e81-f8f9-9439-aadc-f08e2410e401/cluster_a1accffb-3df0-3830-8589-ade335756e45/data/data1/current/BP-1182860866-172.17.0.2-1732071644397 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-20T03:00:58,660 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cc48e81-f8f9-9439-aadc-f08e2410e401/cluster_a1accffb-3df0-3830-8589-ade335756e45/data/data2/current/BP-1182860866-172.17.0.2-1732071644397 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-20T03:00:58,660 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-20T03:00:58,670 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@376fa8f3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-20T03:00:58,671 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@672b88ea{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-20T03:00:58,671 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-20T03:00:58,671 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@76beb053{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-20T03:00:58,671 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@265b24f1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4cc48e81-f8f9-9439-aadc-f08e2410e401/hadoop.log.dir/,STOPPED} 2024-11-20T03:00:58,684 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-20T03:00:58,729 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down